diff --git a/docker/logstash/patterns/amavis b/docker/logstash/patterns/amavis new file mode 100644 index 00000000..07d115b9 --- /dev/null +++ b/docker/logstash/patterns/amavis @@ -0,0 +1,6 @@ +AMAVIS_DIRECTIONALITY ((BouncedInbound|BouncedInternal|DiscardedInbound|NoBounceInternal|RejectedInternal|RelayedInbound|RelayedInternal|RelayedTaggedInbound|Quarantined)(,)?)+ +AMAVIS_STATUS (Passed|Blocked) (CLEAN|MTA-BLOCKED|SPAM|SPAMMY|UNCHECKED(-ENCRYPTED)?|BAD-HEADER-\d|BANNED \(%{DATA:reason}\)|INFECTED \(%{DATA:reason}\)) + +AMAVIS_CHANNEL \(\d+-\d+(-\d+)?\) %{AMAVIS_STATUS:amavis_status} \{%{AMAVIS_DIRECTIONALITY}\}, (?:(ORIGINATING )?LOCAL )?(%{CLIENT_SRC} )?(\[%{IP}\] )?<%{DATA:from}> -> ((<%{DATA:to}>,)+?)? ((Queue-ID|quarantine): %{QUEUEID:local_queueid}, )?(Message-ID: <%{DATA:message_id}>, )?(mail_id: %{DATA}, )?(Hits: %{DATA}, )?(size: %{DATA}, )?(queued_as: %{QUEUEID:remote_queueid}, )?%{DATA} +AMAVIS_MESSAGES (?:%{AMAVIS_CHANNEL}) + diff --git a/docker/logstash/pipeline/13_input_redis.conf b/docker/logstash/pipeline/13_input_redis.conf deleted file mode 100644 index f5106090..00000000 --- a/docker/logstash/pipeline/13_input_redis.conf +++ /dev/null @@ -1,7 +0,0 @@ -# Intended for Redis input from the kolab system -#input { -# redis { -# host => "redis" -# port => 6379 -# } -#} diff --git a/docker/logstash/pipeline/22_filter_maillog.conf b/docker/logstash/pipeline/22_filter_maillog.conf index 192108b8..eda33b5c 100644 --- a/docker/logstash/pipeline/22_filter_maillog.conf +++ b/docker/logstash/pipeline/22_filter_maillog.conf @@ -1,294 +1,301 @@ filter { if [type] == "maillog" { # fingerprint the original message fingerprint { source => [ "message" ] target => "[@metadata][fingerprint]" method => "SHA512" key => "logstash" } mutate { add_field => { "orig_message" => "%{message}" } } # The kolab syslog base matches the RSYSLOG_TraditionalFormat grok { match => { "message" => "%{SYSLOG_BASE}" } patterns_dir => "/usr/share/logstash/patterns/" } # Of which the date is in the ISO8601 format. Note that using # 'date' here sets the Logstash/Elasticsearch timestamp. date { match => [ "syslog_timestamp", "ISO8601" ] } if "_grokparsefailure" not in [tags] { mutate { replace => [ "host", "%{syslog_hostname}" ] replace => [ "message", "%{syslog_message}" ] } mutate { remove_field => [ "path", "syslog_hostname", "syslog_message", "syslog_timestamp" ] } } if [syslog_program] == "ctl_cyrusdb" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "ctl_mboxlist" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "cyr_expire" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "cyr_info" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "imap" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } - # Specialty MyKolab -> Kolab Now item } else if [syslog_program] == "imapkn" { mutate { replace => [ "syslog_program", "cyrus-imapd/imap" ] } - } else if [syslog_program] == "imaps" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } - # Specialty MyKolab -> Kolab Now item } else if [syslog_program] == "imapskn" { mutate { replace => [ "syslog_program", "cyrus-imapd/imaps" ] } - } else if [syslog_program] == "lmtpunix" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "pop3" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "pop3s" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "squatter" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "tls_prune" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } - if [syslog_program] =~ /^cyrus-imapd\// { + if [syslog_program] =~ /^amavis$/ { + if [message] =~ / (Passed|Blocked) / { + grok { + match => [ "message", "%{AMAVIS_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else { + drop { } + } + } + + else if [syslog_program] =~ /^cyrus-imapd\// { if [message] =~ /^getaddrinfo(.*) failed:/ { drop { } } else if [syslog_program] == "cyrus-imapd/ctl_cyrusdb" { grok { match => [ "message", "%{CYRUS_IMAPD_CTL_CYRUSDB_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/ctl_mboxlist" { grok { match => [ "message", "%{CYRUS_IMAPD_CTL_MBOXLIST_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/cyr_expire" { grok { match => [ "message", "%{CYRUS_IMAPD_CYR_EXPIRE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/cyr_info" { grok { match => [ "message", "%{CYRUS_IMAPD_CYR_INFO_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/imap" { grok { match => [ "message", "%{CYRUS_IMAPD_IMAP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/imaps" { grok { match => [ "message", "%{CYRUS_IMAPD_IMAPS_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/lmtpd" { grok { match => [ "message", "%{CYRUS_IMAPD_LMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/lmtpunix" { grok { match => [ "message", "%{CYRUS_IMAPD_LMTPUNIX_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/master" { grok { match => [ "message", "%{CYRUS_IMAPD_MASTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/mupdate" { grok { match => [ "message", "%{CYRUS_IMAPD_MUPDATE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/notifyd" { grok { match => [ "message", "%{CYRUS_IMAPD_NOTIFYD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/pop3" { grok { match => [ "message", "%{CYRUS_IMAPD_POP3_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/pop3s" { grok { match => [ "message", "%{CYRUS_IMAPD_POP3S_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/timsieved" { grok { match => [ "message", "%{CYRUS_IMAPD_SIEVE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/squatter" { grok { match => [ "message", "%{CYRUS_IMAPD_SQUATTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/tls_prune" { grok { match => [ "message", "%{CYRUS_IMAPD_TLS_PRUNE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { mutate { add_tag => [ "_unparsed" ] } } } else if [syslog_program] =~ /^postfix\// { if [message] =~ /fatal:.*table lookup problem/ { drop { } } else if [syslog_program] == "postfix/anvil" { grok { match => [ "message", "%{POSTFIX_ANVIL_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/bounce" { grok { match => [ "message", "%{POSTFIX_BOUNCE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/cleanup" { grok { match => [ "message", "%{POSTFIX_CLEANUP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/error" { grok { match => [ "message", "%{POSTFIX_ERROR_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/lmtp" { grok { match => [ "message", "%{POSTFIX_LMTP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/local" { grok { match => [ "message", "%{POSTFIX_LOCAL_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/master" { grok { match => [ "message", "%{POSTFIX_MASTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/pickup" { grok { match => [ "message", "%{POSTFIX_PICKUP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/postfix-script" { grok { match => [ "message", "%{POSTFIX_SCRIPT_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/qmgr" { grok { match => [ "message", "%{POSTFIX_QMGR_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/scache" { grok { match => [ "message", "%{POSTFIX_SCACHE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/smtp" { grok { match => [ "message", "%{POSTFIX_SMTP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/smtps/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/submission/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/spawn" { grok { match => [ "message", "%{POSTFIX_SPAWN_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/trivial-rewrite" { grok { match => [ "message", "%{POSTFIX_TRIVIAL_REWRITE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { mutate { add_tag => [ "_unparsed" ] } } } else { mutate { add_tag => [ "_unparsed" ] } } } }