diff --git a/docker/logstash/pipeline/22_filter_fingerprint.conf b/docker/logstash/pipeline/22_filter_fingerprint.conf new file mode 100644 index 00000000..52bc38c0 --- /dev/null +++ b/docker/logstash/pipeline/22_filter_fingerprint.conf @@ -0,0 +1,9 @@ +filter { + # fingerprint the original message + fingerprint { + source => [ "message" ] + target => "[@metadata][fingerprint]" + method => "SHA512" + key => "logstash" + } +} diff --git a/docker/logstash/pipeline/23_filter_syslog.conf b/docker/logstash/pipeline/23_filter_syslog.conf new file mode 100644 index 00000000..858f904e --- /dev/null +++ b/docker/logstash/pipeline/23_filter_syslog.conf @@ -0,0 +1,37 @@ +filter { + if [type] == "maillog" { + mutate { + add_field => { "orig_message" => "%{message}" } + } + + # The kolab syslog base matches the RSYSLOG_TraditionalFormat + grok { + match => { + "message" => "%{SYSLOG_BASE}" + } + patterns_dir => "/usr/share/logstash/patterns/" + } + + # Of which the date is in the ISO8601 format. Note that using + # 'date' here sets the Logstash/Elasticsearch timestamp. + date { + match => [ "syslog_timestamp", "ISO8601" ] + } + + if "_grokparsefailure" not in [tags] { + mutate { + replace => [ "host", "%{syslog_hostname}" ] + replace => [ "message", "%{syslog_message}" ] + } + + mutate { + remove_field => [ + "path", + "syslog_hostname", + "syslog_message", + "syslog_timestamp" + ] + } + } + } +} diff --git a/docker/logstash/pipeline/22_filter_maillog.conf b/docker/logstash/pipeline/30_filter_maillog.conf similarity index 90% rename from docker/logstash/pipeline/22_filter_maillog.conf rename to docker/logstash/pipeline/30_filter_maillog.conf index eda33b5c..ec5e8c37 100644 --- a/docker/logstash/pipeline/22_filter_maillog.conf +++ b/docker/logstash/pipeline/30_filter_maillog.conf @@ -1,301 +1,276 @@ filter { if [type] == "maillog" { - # fingerprint the original message - fingerprint { - source => [ "message" ] - target => "[@metadata][fingerprint]" - method => "SHA512" - key => "logstash" - } - - mutate { - add_field => { "orig_message" => "%{message}" } - } - - # The kolab syslog base matches the RSYSLOG_TraditionalFormat - grok { - match => { - "message" => "%{SYSLOG_BASE}" - } - patterns_dir => "/usr/share/logstash/patterns/" - } - - # Of which the date is in the ISO8601 format. Note that using - # 'date' here sets the Logstash/Elasticsearch timestamp. - date { - match => [ "syslog_timestamp", "ISO8601" ] - } - - if "_grokparsefailure" not in [tags] { - mutate { - replace => [ "host", "%{syslog_hostname}" ] - replace => [ "message", "%{syslog_message}" ] - } - - mutate { - remove_field => [ - "path", - "syslog_hostname", - "syslog_message", - "syslog_timestamp" - ] - } - } - if [syslog_program] == "ctl_cyrusdb" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "ctl_mboxlist" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "cyr_expire" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "cyr_info" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "imap" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } # Specialty MyKolab -> Kolab Now item } else if [syslog_program] == "imapkn" { mutate { replace => [ "syslog_program", "cyrus-imapd/imap" ] } } else if [syslog_program] == "imaps" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } # Specialty MyKolab -> Kolab Now item } else if [syslog_program] == "imapskn" { mutate { replace => [ "syslog_program", "cyrus-imapd/imaps" ] } } else if [syslog_program] == "lmtpunix" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "pop3" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "pop3s" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "squatter" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "tls_prune" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } if [syslog_program] =~ /^amavis$/ { if [message] =~ / (Passed|Blocked) / { grok { match => [ "message", "%{AMAVIS_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { drop { } } } else if [syslog_program] =~ /^cyrus-imapd\// { if [message] =~ /^getaddrinfo(.*) failed:/ { drop { } } else if [syslog_program] == "cyrus-imapd/ctl_cyrusdb" { grok { match => [ "message", "%{CYRUS_IMAPD_CTL_CYRUSDB_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/ctl_mboxlist" { grok { match => [ "message", "%{CYRUS_IMAPD_CTL_MBOXLIST_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/cyr_expire" { grok { match => [ "message", "%{CYRUS_IMAPD_CYR_EXPIRE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/cyr_info" { grok { match => [ "message", "%{CYRUS_IMAPD_CYR_INFO_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/imap" { grok { match => [ "message", "%{CYRUS_IMAPD_IMAP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/imaps" { grok { match => [ "message", "%{CYRUS_IMAPD_IMAPS_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/lmtpd" { grok { match => [ "message", "%{CYRUS_IMAPD_LMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/lmtpunix" { grok { match => [ "message", "%{CYRUS_IMAPD_LMTPUNIX_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/master" { grok { match => [ "message", "%{CYRUS_IMAPD_MASTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/mupdate" { grok { match => [ "message", "%{CYRUS_IMAPD_MUPDATE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/notifyd" { grok { match => [ "message", "%{CYRUS_IMAPD_NOTIFYD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/pop3" { grok { match => [ "message", "%{CYRUS_IMAPD_POP3_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/pop3s" { grok { match => [ "message", "%{CYRUS_IMAPD_POP3S_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/timsieved" { grok { match => [ "message", "%{CYRUS_IMAPD_SIEVE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/squatter" { grok { match => [ "message", "%{CYRUS_IMAPD_SQUATTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/tls_prune" { grok { match => [ "message", "%{CYRUS_IMAPD_TLS_PRUNE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { mutate { add_tag => [ "_unparsed" ] } } } + else if [syslog_program] == "opendmarc" { + grok { + match => [ "message", "%{OPENDMARC_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } + else if [syslog_program] =~ /^postfix\// { if [message] =~ /fatal:.*table lookup problem/ { drop { } } else if [syslog_program] == "postfix/anvil" { grok { match => [ "message", "%{POSTFIX_ANVIL_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/bounce" { grok { match => [ "message", "%{POSTFIX_BOUNCE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/cleanup" { grok { match => [ "message", "%{POSTFIX_CLEANUP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/error" { grok { match => [ "message", "%{POSTFIX_ERROR_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/lmtp" { grok { match => [ "message", "%{POSTFIX_LMTP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/local" { grok { match => [ "message", "%{POSTFIX_LOCAL_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/master" { grok { match => [ "message", "%{POSTFIX_MASTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/pickup" { grok { match => [ "message", "%{POSTFIX_PICKUP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/postfix-script" { grok { match => [ "message", "%{POSTFIX_SCRIPT_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/qmgr" { grok { match => [ "message", "%{POSTFIX_QMGR_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/scache" { grok { match => [ "message", "%{POSTFIX_SCACHE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/smtp" { grok { match => [ "message", "%{POSTFIX_SMTP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } + } else if [syslog_program] == "postfix/polite/smtp" { + grok { + match => [ "message", "%{POSTFIX_SMTP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/turtle/smtp" { + grok { + match => [ "message", "%{POSTFIX_SMTP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } } else if [syslog_program] == "postfix/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/smtps/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/submission/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/spawn" { grok { match => [ "message", "%{POSTFIX_SPAWN_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/trivial-rewrite" { grok { match => [ "message", "%{POSTFIX_TRIVIAL_REWRITE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { mutate { add_tag => [ "_unparsed" ] } } } else { mutate { add_tag => [ "_unparsed" ] } } } } diff --git a/docker/logstash/pipeline/30_network.conf b/docker/logstash/pipeline/40_network.conf similarity index 100% rename from docker/logstash/pipeline/30_network.conf rename to docker/logstash/pipeline/40_network.conf diff --git a/docker/logstash/pipeline/31_geoip.conf b/docker/logstash/pipeline/41_geoip.conf similarity index 100% rename from docker/logstash/pipeline/31_geoip.conf rename to docker/logstash/pipeline/41_geoip.conf