diff --git a/apps/kolab_guam/src/rules/kolab_guam_rule_filter_groupware.erl b/apps/kolab_guam/src/rules/kolab_guam_rule_filter_groupware.erl index 7de0eef..51fa790 100644 --- a/apps/kolab_guam/src/rules/kolab_guam_rule_filter_groupware.erl +++ b/apps/kolab_guam/src/rules/kolab_guam_rule_filter_groupware.erl @@ -1,222 +1,258 @@ %% Copyright 2015 Kolab Systems AG (http://www.kolabsys.com) %% %% Aaron Seigo (Kolab Systems) %% %% This program is free software: you can redistribute it and/or modify %% it under the terms of the GNU General Public License as published by %% the Free Software Foundation, either version 3 of the License, or %% (at your option) any later version. %% %% This program is distributed in the hope that it will be useful, %% but WITHOUT ANY WARRANTY; without even the implied warranty of %% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the %% GNU General Public License for more details. %% %% You should have received a copy of the GNU General Public License %% along with this program. If not, see . -module(kolab_guam_rule_filter_groupware). -export([new/1, applies/4, imap_data/3, apply_to_client_message/4, apply_to_server_message/3]). -behavior(kolab_guam_rule). -include("kolab_guam_rule_filter_groupware.hrl"). new(_Config) -> #state { blacklist = undefined }. applies(_ConnectionDetails, _Buffer, { _Tag, Command, Data }, State) -> Applies = apply_if_id_matches(Command, Data, State), %lager:debug("********** Checking ...~n Command: ~s ~s, Result ~p", [Command, Data, Applies]), { Applies, State }. apply_to_client_message(_ImapSession, Buffer, undefined, State) -> { Buffer, State }; apply_to_client_message(ImapSession, Buffer, { Tag, Command, Data }, State) -> { Active, StateTag } = case is_triggering_command(Command, Data, State) of true -> fetch_metadata(ImapSession, State), { true, Tag }; _ -> { false, <<>> } end, %lager:info("Client sent: ~s ~s ~p", [Command, Data, Active]), { Buffer, State#state{ active = Active, tag = StateTag }}. apply_to_server_message(_ImapSession, Buffer, #state{ active = true } = State) -> filter_folders(Buffer, State); apply_to_server_message(_ImapSession, Buffer, State) -> { Buffer, State }. imap_data(_ResponseToken, { error, _Reason }, State) -> State; imap_data(_ResponseToken, Response, State) -> %TODO: we don't need Foo/Bar if we already have Foo, so filter folders-of-groupwarefolders Blacklist = lists:foldl(fun({ _Folder, [ { _Property, null } ]}, Acc) -> Acc; ({ _Folder, [ { _Property, <<"mail", _Rest/binary>> } ]}, Acc) -> Acc; ({ Folder, _ }, Acc) -> [{ Folder, <> }|Acc] end, [], Response), State#state{ blacklist = Blacklist }. %%PRIVATE is_triggering_command(Command, Data, #state{ trigger_commands = TriggerCommands }) -> %% if the command is in the list of trigger commands and the ending is not "" (which means "send me %% the root and separator" according to RFC 3501), then it is treated as a triggering event lists:any(fun(T) -> (Command =:= T) andalso (binary:longest_common_suffix([Data, <<"\"\"">>]) =/= 2) end, TriggerCommands). fetch_metadata(none, #state{ blacklist = undefined }) -> ok; fetch_metadata(ImapSession, #state{ blacklist = undefined }) -> eimap:get_folder_metadata(ImapSession, self(), { rule_data, ?MODULE, blacklist }, "*", ["/shared/vendor/kolab/folder-type"]); fetch_metadata(_ImapSession, _State) -> ok. apply_if_id_matches(<<"ID">>, Data, _State) -> apply_if_found_kolab(string:str(string:to_lower(binary_to_list(Data)), "/kolab")); apply_if_id_matches(Command, Data, State) -> case is_triggering_command(Command, Data, State) of true -> true; _ -> notyet end. apply_if_found_kolab(0) -> true; apply_if_found_kolab(_) -> false. -possibly_append_newline(<<>>) -> <<>> ; -possibly_append_newline(Response) -> <> . +split(Lines, Data, Pattern) -> + case binary:match(Data, Pattern) of + nomatch -> {lists:reverse(Lines), Data}; + { Start, Length } -> + Num = Start + Length, + <> = Data, + split([Line|Lines], Remainder, Pattern) + end. +split(Lines, Data) -> + split(Lines, Data, binary:compile_pattern(<<"\r\n">>)). + +% Split Data into a list of lines separated by \r\n (each entry including the \r\n) +split_lines(Data) -> + split([], Data). + +% Literal complete +connect_next_line(Line, RemainingLines, 0) -> + {Line, RemainingLines, ok}; +% Not enough lines to complete the literal +connect_next_line(Line, [], _ContinuationBytes) -> + {Line, [], nok}; +% Connecting the next line +connect_next_line(Line, [NextLine|RemainingLines], ContinuationBytes) -> + connect_next_line(<>, RemainingLines, ContinuationBytes - (byte_size(NextLine)-2)). + +connect_literals([], Acc) -> + {lists:reverse(Acc), []}; +connect_literals([Line|RemainingLines], Acc) -> + { _StrippedNextLine, ContinuationBytes } = eimap_utils:num_literal_continuation_bytes(binary:part(Line, 0, byte_size(Line)-2)), + {FullLine, RemainingLinesAfterConnect, Result} = case ContinuationBytes of + none -> {Line, RemainingLines, ok}; + _ -> connect_next_line(Line, RemainingLines, ContinuationBytes) + end, + case Result of + nok -> {Acc, [Line|RemainingLines]}; + ok -> connect_literals(RemainingLinesAfterConnect, [FullLine|Acc]) + end. + filter_folders(<<>>, State) -> { <<>>, State#state{ active = true } }; filter_folders(Buffer, #state{ last_chunk = LeftOvers } = State) -> % Add the left overs from the previous buffer to the current buffer FullBuffer = <>, - % From that buffer, only take the complete lines and save off the remainder. - { FullLinesBuffer, LastChunk } = eimap_utils:only_full_lines(FullBuffer), - % Create a list so we can filter the individual folders - ListResponses = binary:split(FullLinesBuffer, <<"\r\n">>, [ global ]), - { Response, More } = filter_folders(State, ListResponses, { <<>>, true }), - %io:format("Filtered ... ~p~n", [Response]), - %Note that the last list item does not contain \r\n, so that needs to be added unless we filtered the complete content. - { possibly_append_newline(Response), State#state { active = More, last_chunk = LastChunk } }. + + {Lines, LastChunk} = split_lines(FullBuffer), + {FullLines, RemainingLines} = connect_literals(Lines, []), + + { Response, More } = filter_folders(State, FullLines, { <<>>, true }), + + Joined = list_to_binary(RemainingLines), + {<>, State#state{ active = More, last_chunk = <>}}. filter_folders(_State, [], Return) -> Return; filter_folders(_State, _Folders, { Acc, false }) -> { Acc, false }; filter_folders(State, [Unfiltered|Folders], { Acc, _More }) -> filter_folders(State, Folders, filter_folder(State, Unfiltered, Acc)). filter_folder(_State, <<>>, Acc) -> { Acc, true }; filter_folder(State, <<"* LIST ", Details/binary>> = Response, Acc) -> { filter_on_details(State, Response, Acc, Details), true }; filter_folder(State, <<"* XLIST ", Details/binary>> = Response, Acc) -> { filter_on_details(State, Response, Acc, Details), true }; filter_folder(State, <<"* LSUB ", Details/binary>> = Response, Acc) -> { filter_on_details(State, Response, Acc, Details), true }; filter_folder(State, <<"* STATUS ", Details/binary>> = Response, Acc) -> { filter_on_details(State, Response, Acc, Details), true }; filter_folder(#state{ tag = Tag }, Response, Acc) -> HasMore = case byte_size(Tag) =< byte_size(Response) of true -> case binary:match(Response, Tag, [{ scope, { 0, byte_size(Tag) } }]) of nomatch -> true; _ -> false % we have found our closing tag! end; false -> true end, { add_response(Response, Acc), HasMore }. filter_on_details(#state{ blacklist = Blacklist }, Response, Acc, Details) -> %% Remove "*" and extract response command name { _, Start, _ } = pop_token(Response), %% asterisk { Cmd, _, _ } = pop_token(Start), %% command %% Extract folder name Suffix = case Cmd =:= <<"STATUS">> of true -> Details; _ -> { Pos, _Length } = binary:match(Details, [<<")">>], []), { _Delimiter, Rest, _} = pop_token(binary:part(Details, Pos + 2, byte_size(Details) - Pos - 2)), Rest end, - { Folder, _, _ } = pop_token(list_to_binary([Suffix, <<"\r\n">>])), + { Folder, _, _ } = pop_token(Suffix), %% Check the folder in blacklist %% io:format("COMPARING ~p ??? ~p~n", [Folder, in_blacklist(Folder, Blacklist)]), case in_blacklist(Folder, Blacklist) of true -> Acc; _ -> add_response(Response, Acc) end. add_response(Response, <<>>) -> Response; -add_response(Response, Acc) -> <>. +add_response(Response, Acc) -> <>. in_blacklist(_Folder, undefined) -> false; in_blacklist(_Folder, []) -> false; in_blacklist(Folder, [{ Literal, Prefix }|List]) -> case Literal =:= Folder of true -> true; _ -> case binary:match(Folder, Prefix) of { 0, _ } -> true; _ -> in_blacklist(Folder, List) end end. %% pop_token from https://github.com/MainframeHQ/switchboard/blob/master/src/imap.erl (BSD Lic.) %% with some small changes by Aleksander Machniak pop_token(Data) -> pop_token(Data, none). pop_token(<<>>, State) -> {none, <<>>, State}; %% Consume hanging spaces pop_token(<<" ", Rest/binary>>, none) -> pop_token(Rest, none); %% \r\n pop_token(<<$\r, $\n, Rest/binary>>, none) -> {crlf, Rest, none}; %% NIL pop_token(<<"NIL", Rest/binary>>, none) -> {nil, Rest, none}; %% ( | ) | [ | ] pop_token(<<$(, Rest/binary>>, none) -> {'(', Rest, none}; pop_token(<<$), Rest/binary>>, none) -> {')', Rest, none}; %% Atom pop_token(<> = Data, {atom, AtomAcc}) when C =:= 32; C =:= 40; C =:= 41; C =:= $(; C =:= $) -> {AtomAcc, Data, none}; pop_token(<<$\r, $\n, _/binary>> = Data, {atom, AtomAcc}) -> {AtomAcc, Data, none}; pop_token(<>, none) when C >= 35, C < 123 -> pop_token(Rest, {atom, <>}); pop_token(<>, {atom, AtomAcc}) when C >= 35, C < 123 -> pop_token(Rest, {atom, <>}); %% Literal Strings pop_token(<<${, Rest/binary>>, none) -> pop_token(Rest, {literal, <<>>}); pop_token(<<$}, $\r, $\n, Rest/binary>>, {literal, ByteAcc}) -> pop_token(Rest, {literal, binary_to_integer(ByteAcc), <<>>}); pop_token(<>, {literal, ByteAcc}) when D >= 48, D < 58 -> pop_token(Rest, {literal, <>}); pop_token(Binary, {literal, Bytes, LiteralAcc}) when is_integer(Bytes) -> case Binary of <> -> {<>, Rest, none}; _ -> %% If the binary is too short, accumulate it in the state pop_token(<<>>, {literal, Bytes - size(Binary), <>}) end; %% Quoted Strings pop_token(<<$", Rest/binary>>, none) -> pop_token(Rest, {quoted, <<>>}); pop_token(<<$\\, C, Rest/binary>>, {quoted, Acc}) -> pop_token(Rest, {quoted, <>}); pop_token(<<$", Rest/binary>>, {quoted, Acc}) -> {Acc, Rest, none}; pop_token(<<$\r, $\n, _>>, {quoted, _}) -> throw({error, crlf_in_quoted}); pop_token(<>, {quoted, Acc}) -> pop_token(Rest, {quoted, <>}); pop_token(Binary, _) -> {none, Binary, none}. diff --git a/apps/kolab_guam/test/kolab_guam_rules_SUITE.erl b/apps/kolab_guam/test/kolab_guam_rules_SUITE.erl index a9a660e..bbb1180 100644 --- a/apps/kolab_guam/test/kolab_guam_rules_SUITE.erl +++ b/apps/kolab_guam/test/kolab_guam_rules_SUITE.erl @@ -1,217 +1,218 @@ %% Copyright 2015 Kolab Systems AG (http://www.kolabsys.com) %% %% Aaron Seigo (Kolab Systems) %% %% This program is free software: you can redistribute it and/or modify %% it under the terms of the GNU General Public License as published by %% the Free Software Foundation, either version 3 of the License, or %% (at your option) any later version. %% %% This program is distributed in the hope that it will be useful, %% but WITHOUT ANY WARRANTY; without even the implied warranty of %% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the %% GNU General Public License for more details. %% %% You should have received a copy of the GNU General Public License %% along with this program. If not, see . -module(kolab_guam_rules_SUITE). % easier than exporting by name -compile(export_all). % required for common_test to work -include_lib("common_test/include/ct.hrl"). -include("../src/rules/kolab_guam_rule_filter_groupware.hrl"). %%%%%%%%%%%%%%%%%%%%%%%%%%% %% common test callbacks %% %%%%%%%%%%%%%%%%%%%%%%%%%%% % Specify a list of all unit test functions all() -> [ kolab_guam_rule_filter_groupware_responsefiltering_test, kolab_guam_rule_filter_groupware_responsefiltering_multipacket_test ]. % required, but can just return Config. this is a suite level setup function. init_per_suite(Config) -> Config. % required, but can just return Config. this is a suite level tear down function. end_per_suite(Config) -> Config. % optional, can do function level setup for all functions, % or for individual functions by matching on TestCase. init_per_testcase(_TestCase, Config) -> Config. % optional, can do function level tear down for all functions, % or for individual functions by matching on TestCase. end_per_testcase(_TestCase, Config) -> Config. % c("apps/kolab_guam/test/kolab_guam_sup_tests.erl"). eunit:test(kolab_guam_sup_tests). kolab_guam_rule_filter_groupware_responsefiltering_test(_TestConfig) -> %% Data to be fed into the test, one tuple per iteration %% Tuple format: { [] = folder_blacklist, input_data, correct_output } Data = [ { [ {<<"Calendar">>, <<"Calendar/">>}, {<<"Calendar/Personal Calendar">>, <<"Calendar/Personal Calendar/">>}, {<<"Configuration">>, <<"Configuration/">>}, {<<"Contacts">>, <<"Contacts/">>}, {<<"Contacts/Personal Contacts">>, <<"Contacts/Personal Contacts/">>}, {<<"Files">>, <<"Files/">>}, {<<"Journal">>, <<"Journal/">>}, {<<"Notes">>, <<"Notes/">>}, {<<"Tasks">>, <<"Tasks/">>} ], <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n* LIST (\\Subscribed) \"/\" Archive\r\n* LIST (\\Subscribed \\HasChildren) \"/\" Calendar\r\n* LIST (\\Subscribed) \"/\" \"Calendar/Personal Calendar\"\r\n* LIST (\\Subscribed) \"/\" Configuration\r\n* LIST (\\Subscribed \\HasChildren) \"/\" Contacts\r\n* LIST (\\Subscribed) \"/\" \"Contacts/Personal Contacts\"\r\n* LIST (\\Subscribed) \"/\" Drafts\r\n* LIST (\\Subscribed) \"/\" Files\r\n* LIST (\\Subscribed) \"/\" Journal\r\n* LIST (\\Subscribed) \"/\" Notes\r\n* LIST (\\Subscribed) \"/\" Sent\r\n* LIST (\\Subscribed) \"/\" Spam\r\n* LIST (\\Subscribed) \"/\" Tasks\r\n* LIST (\\Subscribed) \"/\" Trash\r\n7 OK Completed (0.000 secs 15 calls)\r\n">>, <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n* LIST (\\Subscribed) \"/\" Archive\r\n* LIST (\\Subscribed) \"/\" Drafts\r\n* LIST (\\Subscribed) \"/\" Sent\r\n* LIST (\\Subscribed) \"/\" Spam\r\n* LIST (\\Subscribed) \"/\" Trash\r\n7 OK Completed (0.000 secs 15 calls)\r\n">> } ], %% setup boilerplate Config = {}, State = kolab_guam_rule_filter_groupware:new(Config), ServerConfig = kolab_guam_sup:default_imap_server_config(), { ok, ImapSession } = eimap:start_link(ServerConfig), %% create the rule, ready for testing ClientData = <<"7 list (subscribed) \"\" \"*\" return (special-use)">>, Split = eimap_utils:split_command_into_components(ClientData), { _, ReadyState } = kolab_guam_rule_filter_groupware:apply_to_client_message(ImapSession, ClientData, Split, State), %% run the dataset through the rule lists:foreach(fun({ Blacklist, Input, Filtered }) -> BlacklistState = ReadyState#state{ blacklist = Blacklist }, { Filtered, _NewState } = kolab_guam_rule_filter_groupware:apply_to_server_message(ImapSession, Input, BlacklistState) end, Data). kolab_guam_rule_filter_groupware_responsefiltering_multipacket_test(_TestConfig) -> %% Data to be fed into the test, one tuple per iteration %% Tuple format: { [] = folder_blacklist, [] = input_data_packets, correct_output } Data = [ { [ {<<"Calendar">>, <<"Calendar/">>}, {<<"Calendar/Personal Calendar">>, <<"Calendar/Personal Calendar/">>}, {<<"Configuration">>, <<"Configuration/">>}, {<<"Contacts">>, <<"Contacts/">>}, {<<"Contacts/Personal Contacts">>, <<"Contacts/Personal Contacts/">>}, {<<"Files">>, <<"Files/">>}, {<<"Journal">>, <<"Journal/">>}, {<<"Notes">>, <<"Notes/">>}, {<<"Tasks">>, <<"Tasks/">>} ], [ <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n* LIST (\\Subscribed) \"/\" Archive\r\n* LIST (\\Subscribed \\HasChildren) \"/\" Calendar\r\n* LIST (\\Subscribed) \"/\" \"Calendar/Personal Calendar\"\r\n* LIST (\\Subscribed) \"/\" Configuration\r\n* LIST (\\Subscribed \\HasChildren) \"/\" Contacts\r\n* LIST (\\Subscribed) \"/\" \"Contacts/Personal Contacts\"\r\n* LIST (\\Subscribed) \"/\" Drafts\r\n* LIST (\\Subscribed) \"/\" Files\r\n* LIST (\\Subscribed) \"/\" Journal\r\n* LIST (\\Subscribed)">>, <<" \"/\" Notes\r\n* LIST (\\Subscribed) \"/\" Sent\r\n* LIST (\\Subscribed) \"/\" Spam\r\n* LIST (\\Subscribed) \"/\" Tasks\r\n* LIST (\\Subscribed) \"/\" Trash\r\n7 OK Completed (0.000 secs 15 calls)\r\n">> ], <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n* LIST (\\Subscribed) \"/\" Archive\r\n* LIST (\\Subscribed) \"/\" Drafts\r\n* LIST (\\Subscribed) \"/\" Sent\r\n* LIST (\\Subscribed) \"/\" Spam\r\n* LIST (\\Subscribed) \"/\" Trash\r\n7 OK Completed (0.000 secs 15 calls)\r\n">> }, %Filter complete packet { [ {<<"Calendar">>, <<"Calendar/">>} ], [ <<"* LIST (\\Subscribed \\HasChildren) \"/\" Calendar\r\n">>, <<"7 OK Completed (0.000 secs 15 calls)\r\n">> ], <<"7 OK Completed (0.000 secs 15 calls)\r\n">> }, %Test folder names with square brackes { [ {<<"[Calendar]">>,<<"[Calendar]/">>} ], [ <<"* LSUB () \".\" INBOX\r\n* LSUB () \".\" \"[Stuff].Sent Mail.Sent\"\r\n* LSUB () \".\" [Calendar]\r\nwgku OK Lsub completed (0.001 + 0.000 secs).\r\n">> ], <<"* LSUB () \".\" INBOX\r\n* LSUB () \".\" \"[Stuff].Sent Mail.Sent\"\r\nwgku OK Lsub completed (0.001 + 0.000 secs).\r\n">> }, %Split CR and LF { [ {<<"Calendar">>, <<"Calendar/">>} ], [ <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r">>, <<"\n* LIST (\\Subscribed \\HasChildren) \"/\" Calendar\r">>, <<"\n7 OK Completed (0.000 secs 15 calls)\r\n">> ], <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n7 OK Completed (0.000 secs 15 calls)\r\n">> }, %Numeric folders { [ ], [ <<"* LIST () \"/\" 2017\r\n">>, <<"7 OK Completed (0.000 secs 15 calls)\r\n">> ], <<"* LIST () \"/\" 2017\r\n7 OK Completed (0.000 secs 15 calls)\r\n">> }, %LIST-EXTENDED + LIST-STATUS response (as used by evolution) { [ {<<"Calendar">>, <<"Calendar/">>}, {<<"Calendar/Calendar2/">>, <<"Calendar/Calendar2/">>} ], [ <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n">>, <<"* STATUS \"INBOX\" (MESSAGES 17 UNSEEN 16)\r\n">>, <<"* LIST (\\Subscribed \\HasChildren) \"/\" Calendar (CHILDINFO (\"SUBSCRIBED\"))\r\n">>, <<"* STATUS \"Calendar\" (MESSAGES 17 UNSEEN 16)\r\n">>, <<"* LIST (\\Subscribed \\HasNoChildren) \"/\" Calendar/Calendar2\r\n">>, <<"* STATUS \"Calendar/Calendar2\" (MESSAGES 17 UNSEEN 16)\r\n">>, <<"7 OK Completed (0.000 secs 15 calls)\r\n">> ], <<"* LIST (\\Noinferiors \\Subscribed) \"/\" INBOX\r\n* STATUS \"INBOX\" (MESSAGES 17 UNSEEN 16)\r\n7 OK Completed (0.000 secs 15 calls)\r\n">> }, %String literals - %FIXME Filtering on string literals is broken, it will attempt to filter on {8}Calendar - %It doesn't currently matter though, because there's no literals in folder names (not normally anyways) { [ - %{<<"Calendar">>, <<"Calendar/">>} + {<<"Calendar">>, <<"Calendar/">>}, + {<<"Configuration">>, <<"Configuration/">>} ], [ - %<<"* LIST (\Subscribed) \"/\" {8}Calendar\r\n">>, + <<"* LIST (\Subscribed) \"/\" {8}\r\nCalendar\r\n">>, + <<"* LIST (\Subscribed) \"/\" {13}\r\n">>, + <<"Configuration\r\n">>, <<"* LIST (\Subscribed) \"/\" {6}\r\n">>, <<"Folder\r\n">> ], <<"* LIST (\Subscribed) \"/\" {6}\r\nFolder\r\n">> } ], %% setup boilerplate Config = {}, State = kolab_guam_rule_filter_groupware:new(Config), ServerConfig = kolab_guam_sup:default_imap_server_config(), { ok, ImapSession } = eimap:start_link(ServerConfig), %% create the rule, ready for testing ClientData = <<"7 list (subscribed) \"\" \"*\" return (special-use)">>, Split = eimap_utils:split_command_into_components(ClientData), { _, ReadyState } = kolab_guam_rule_filter_groupware:apply_to_client_message(ImapSession, ClientData, Split, State), %% run the dataset through the rule lists:foreach(fun({ Blacklist, Input, Filtered }) -> BlacklistState = ReadyState#state{ blacklist = Blacklist }, Filtered = filter_groupware_packets(ImapSession, BlacklistState, Input, <<>>) end, Data). filter_groupware_packets(_ImapSession, _ReadyState, [], Buffer) -> Buffer; filter_groupware_packets(ImapSession, ReadyState, [Input|More], Buffer) -> { Processed, State } = kolab_guam_rule_filter_groupware:apply_to_server_message(ImapSession, Input, ReadyState), filter_groupware_packets(ImapSession, State, More, <>).