diff --git a/docker/logstash/_grokparsefailures/.gitignore b/docker/logstash/_grokparsefailures/.gitignore new file mode 100644 index 00000000..8f9745a4 --- /dev/null +++ b/docker/logstash/_grokparsefailures/.gitignore @@ -0,0 +1,2 @@ +*.json +*.log diff --git a/docker/logstash/patterns/base b/docker/logstash/patterns/base new file mode 100644 index 00000000..edcd8073 --- /dev/null +++ b/docker/logstash/patterns/base @@ -0,0 +1,32 @@ +DATA .*? +GREEDYDATA .* + +# RFC 5322 at http://tools.ietf.org/html/rfc5322#section-3.4 +EMAIL_ADDRESS_LOCAL (?:[a-zA-Z0-9!#\$%&'\*\+/=\?\^_`{\|}~\-]+(?:\.[a-zA-Z0-9!#\$%&'\*\+/=\?\^_`{\|}~\-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*") +EMAIL_ADDRESS_DOMAIN (?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-zA-Z0-9-]*[a-zA-Z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]) +EMAIL_ADDRESS %{EMAIL_ADDRESS_LOCAL}(?:@%{EMAIL_ADDRESS_DOMAIN:domain})? + +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?\d\d){1,2} +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHDAY (?:(?:(0|\s)[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) + +POSINT \b(?:[1-9][0-9]*)\b +NONNEGINT \b(?:[0-9]+)\b + +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? + +# A client address as listed in Postfix or Cyrus IMAP messages +CLIENT ((?:(%{HOSTNAME:client_host}|%{IP_OR_HOST:client_host}|unknown)\s?)?)?\[(?:%{IP:src_ip}|unknown)?\](?::%{POSINT})? +CLIENT_DEST ((?:(%{HOSTNAME:client_host}|%{IP_OR_HOST:client_host}|unknown)\s?)?)?\[(?:%{IP:dest_ip}|unknown)?\](?::%{POSINT})? +CLIENT_SRC ((?:(%{HOSTNAME:client_host}|%{IP_OR_HOST:client_host}|unknown)\s?)?)?\[(?:%{IP:src_ip}|unknown)?\](?::%{POSINT})? + diff --git a/docker/logstash/patterns/cyrus-imapd b/docker/logstash/patterns/cyrus-imapd new file mode 100644 index 00000000..05d37129 --- /dev/null +++ b/docker/logstash/patterns/cyrus-imapd @@ -0,0 +1,85 @@ +CYRUS_IMAP_SESSIONID (?:%{IP_OR_HOST}|%{DATA})-\d+-\d+-\d+(-\d+)? +BADLOGIN_REASONS \[?(SASL\(-13\): )?(authentication failure: checkpass failed|user not found: could not find password|authentication failure: Password verification failed|authentication failure)\]? + +CYRUS_DUMMY_FAIL fail me + +CYRUS_IMAPD_AUDIT_ACTIONS (?:append|discard|duplicate|modseq|proxy|redirect|traffic) +CYRUS_IMAPD_SIEVE_ACTIONS (discarded|redirected|rejected) + +CYRUS_IMAPD_AUDITLOG auditlog: %{CYRUS_IMAPD_AUDIT_ACTIONS} (?:%{EMAIL_ADDRESS:client_auth} )?(?:sessionid=<%{CYRUS_IMAP_SESSIONID:session_id}> )?(?:action=<%{CYRUS_IMAPD_AUDIT_ACTIONS}> )?(?:bytes_in=<%{BASE10NUM:bytes_in}> bytes_out=<%{BASE10NUM:bytes_out}>)?(?:message-id=<%{DATA:message_id}>(?:-%{EMAIL_ADDRESS:client_auth} user=<%{DATA}> date=<%{DATA}>)?(?:mailbox=<%{DATA:mailbox}> uniqueid=<%{DATA:uniqueid}>(?: highestmodseq=<%{BASE10NUM:modseq}>|uid=<%{BASE10NUM}> guid=<%{DATA}> cid=<%{DATA}>)?)?)? +CYRUS_IMAPD_CHECKING_DATABASES (?:checkpointing cyrus databases|(?:done )?(?:checkpointing|recovering) cyrus databases) +CYRUS_IMAPD_CLIENTID client id: (?:(?:"name" "%{DATA:client_name}" "version" "%{DATA:client_version}") )?%{GREEDYDATA:client_extra} + +CYRUS_IMAPD_DELPRUNE Removed \d+ deleted mailboxes +CYRUS_IMAPD_DUPELIM dupelim: eliminated duplicate message to %{DATA:mailbox} id <%{DATA:message_id}>(?:-%{EMAIL_ADDRESS:client_auth})? date %{DATESTAMP_RFC2822} (\(%{DATA}\) )?\((?:delivery|redirect)\) +CYRUS_IMAPD_DUPPRUNE duplicate_prune: %{GREEDYDATA} +CYRUS_IMAPD_EVENT EVENT, , , , %{GREEDYDATA:event_json} +CYRUS_IMAPD_EXPUNGE Expunged \d+ (out of \d+ )?messages from (?:%{DATA:mailbox}|\d+ mailboxes) +CYRUS_IMAPD_INDEX_UPGRADE Index upgrade: %{DATA:mailbox} \(\d+ -> \d+\) +CYRUS_IMAPD_LMTPUNIX_DELIVERY Delivered:\s+(<)?%{DATA:message_id(>)? to mailbox: %{GREEDYDATA:mailbox} +CYRUS_IMAPD_LMTPUNIX_LONGLOCK (mailbox|skiplist): longlock %{DATA} for [0-9]+\.[0-9]+ seconds +CYRUS_IMAPD_LMTPUNIX_SIEVE (?:%{CYRUS_IMAPD_SIEVE_ACTION}|%{CYRUS_IMAPD_SIEVE_RUNTIME_ERROR}) +# Note that sometimes a reverse DNS record is simply a '.', because it is virtually unrestricted. +CYRUS_IMAPD_LOGIN (?:bad)?login: (?:%{CLIENT}|\.) (?:%{WORD:mech} )?(?:(\(<)?%{EMAIL_ADDRESS:client_auth}(>\))? )?(?:%{WORD:mech}(\+TLS)? )?(?:(User logged in( SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>)?)|(%{BADLOGIN_REASONS:reason})?) +CYRUS_IMAPD_MAILBOX_DELETE Deleted mailbox %{DATA:mailbox} +CYRUS_IMAPD_MAILBOX_OPEN open: user %{EMAIL_ADDRESS:client_auth} opened %{DATA:mailbox} +CYRUS_IMAPD_MAILBOX_EVENT (indexing|Deleted|Repacking) mailbox %{DATA:mailbox}(\.\.\. )? +CYRUS_IMAPD_MAILBOX_RENAME Rename: %{DATA:mailbox} -> %{DATA:mailbox} +CYRUS_IMAPD_MASTER_STATUS (?:service %{WORD} pid \d+ in %{WORD} state: terminated abnormally|(?:process type:SERVICE name:%{NOTSPACE} path:%{PATH} age:%{POSREAL}s pid:%{POSINT} )?(?:exiting|exited)(?:, status %{BASE10NUM})?) +CYRUS_IMAPD_NETWORK_ERROR (Fatal error: Lost connection to selected backend|(?:Connection (?:timed out|reset by peer)|Error decompressing data|idle for too long|Invalid argument|No route to host), closing connection) +CYRUS_IMAPD_NONOTIFY unable to sendto\(\) notify socket: No such file or directory +CYRUS_IMAPD_POP_COUNTS counts: retr=<\d+> top=<\d+> dele=<\d+> +CYRUS_IMAPD_POP_FAILED pop3(s)? failed: %{CLIENT} +CYRUS_IMAPD_PROXY proxy %{EMAIL_ADDRESS:client_auth} session-id=<%{CYRUS_IMAP_SESSIONID:session_id}> remote=<%{CYRUS_IMAP_SESSIONID:session_id}> +CYRUS_IMAPD_SAME_MSG %{DATA:mailbox}: same message appears twice %{INT} %{INT} +CYRUS_IMAPD_SESSION_STATS (?:IMAP|LMTP) session stats : I/O read : %{BASE10NUM:bytes_read} bytes : I/O write : %{BASE10NUM:bytes_write} bytes +CYRUS_IMAPD_SETRLIMIT (?:setrlimit: Unable to set file descriptors limit to -1: Operation not permitted|retrying with \d+ \(current max\)) +CYRUS_IMAPD_SETSOCKETOPT unable to setsocketopt\(IP_TOS\)(?: service %{DATA})?: Operation not supported +CYRUS_IMAPD_SHUTDOWN (All children have exited, closing down|attempting clean shutdown on signal|graceful shutdown) +CYRUS_IMAPD_SIEVE_ACTION sieve %{CYRUS_IMAPD_SIEVE_ACTIONS}: <%{DATA:message_id}>(?: to: %{EMAIL_ADDRESS:to})? +CYRUS_IMAPD_SIEVE_DISCONNECT Lost connection to client -- exiting +CYRUS_IMAPD_SIEVE_RUNTIME_ERROR sieve runtime error for %{EMAIL_ADDRESS:client_auth} id <%{DATA:message_id}>: Fileinto: Mailbox does not exist +CYRUS_IMAPD_SKIPLIST skiplist: (checkpointed %{PATH:db_path} \(%{INT:db_records} records, %{INT:db_bytes} bytes\) in %{POSREAL:db_seconds} sec(ond(s)?)?|clean shutdown file missing, updating recovery stamp) +CYRUS_IMAPD_UNUSED_CACHE Removed unused cache file %{PATH} +CYRUS_IMAPD_USAGE USAGE %{EMAIL_ADDRESS:client_auth} user: %{POSREAL} sys: %{POSREAL} + +CYRUS_IMAPD_MAILBOX_EVENTS (%{CYRUS_IMAPD_MAILBOX_DELETE}|%{CYRUS_IMAPD_MAILBOX_OPEN}|%{CYRUS_IMAPD_MAILBOX_EVENT}|%{CYRUS_IMAPD_MAILBOX_RENAME}) +CYRUS_IMAPD_ERRORS (?:%{CYRUS_IMAPD_IOERRORS}|%{CYRUS_IMAPD_NETWORK_ERROR}|%{CYRUS_IMAPD_NONOTIFY}) +CYRUS_IMAPD_MASTER_ERRORS (?:%{CYRUS_IMAPD_NONOTIFY}|%{CYRUS_IMAPD_SETRLIMIT}|%{CYRUS_IMAPD_SETSOCKETOPT}|%{CYRUS_IMAPD_SHUTDOWN}) + +CYRUS_CERT_EXPIRED cert has expired +CYRUS_CHECKPOINT checkpointing cyrus databases +# <22>May 24 13:30:08 kolab11 cyrus-imapd/imap[19410]: client id: "name" "Roundcube" "version" "1.0.0" "php" "5.3.27" "os" "Linux" "command" "/?_task=mail&_refresh=1&_mbox=INBOX&_action=list&_remote=1&_unlock=loading1400931009534&_=1400924844797" +CYRUS_CONNECT accepted connection +CYRUS_IMAPD_DHPARAMS ^inittls: Loading (hard-coded DH parameters|DH parameters from file)$ +CYRUS_EXEC (?:about to exec %{DATA}|executed) +CYRUS_EXIT process %{BASE10NUM:pid} exited, status %{BASE10NUM:pid_exit} +CYRUS_EXPIRE unable to open mailbox %{DATA:mailbox} +CYRUS_IDLED error sending to idled: %{BASE10NUM} +CYRUS_IOERROR_INDEX IOERROR: (?:opening index %{DATA:mailbox}: %{DATA:reason}|%{GREEDYDATA:reason}) +CYRUS_IOERROR_MESSAGE IOERROR: reading message: %{DATA:reason} +CYRUS_QRESYNC inefficient qresync \(%{POSINT} > %{POSINT}\) %{DATA:mailbox} +CYRUS_SQUATTER (?:(done )?indexing mailboxes|error opening %{DATA:mailbox}: %{DATA:reason}|timezone shift for squatter - altering schedule by %{BASE10NUM} seconds|SQUAT returned \d+ messages) +CYRUS_IMAPD_IOERRORS (%{CYRUS_IOERROR_INDEX}|%{CYRUS_IOERROR_MESSAGE}) + +CYRUS_SSLACCEPT SSL_accept\(\) %{WORD} -> %{WORD} +CYRUS_STARTTLS starttls: %{DATA:tls_version} with cipher %{DATA:tls_cipher} \(\d+/\d+ bits new\) no authentication +CYRUS_TLSERROR (?:Fatal error: (?:Lost connection to input stream|tls_start_servertls\(\) failed)|verify error:num=(?:18:self signed certificate|10:certificate has expired)) +CYRUS_TLSNEG (?:(\[pop3d\] )?(imaps TLS|STARTTLS)) (negotiation )?failed: %{CLIENT} +CYRUS_TLSPRUNE tls_prune: purged \d+ out of \d+ entries +CYRUS_IMAPD_SSLTLS (%{CYRUS_SSLACCEPT}|%{CYRUS_STARTTLS}|%{CYRUS_TLSERROR}|%{CYRUS_TLSNEG}|%{CYRUS_TLSPRUNE}) + +CYRUS_IMAPD_CTL_CYRUSDB_MESSAGES (?:%{CYRUS_IMAPD_CHECKING_DATABASES}|%{CYRUS_IMAPD_SKIPLIST}) +CYRUS_IMAPD_CYR_EXPIRE_MESSAGES (?:%{CYRUS_IMAPD_DELPRUNE}|%{CYRUS_IMAPD_DUPPRUNE}) +CYRUS_IMAPD_IMAP_MESSAGES (?:%{CYRUS_IMAPD_AUDITLOG}|%{CYRUS_IMAPD_CLIENTID}|%{CYRUS_IMAPD_DHPARAMS}|%{CYRUS_IMAPD_EXPUNGE}|%{CYRUS_IMAPD_ERRORS}|%{CYRUS_IMAPD_INDEX_UPGRADE}|%{CYRUS_IMAPD_LOGIN}|%{CYRUS_IMAPD_MAILBOX_EVENTS}|%{CYRUS_IMAPD_PROXY}|%{CYRUS_IMAPD_SAME_MSG}|%{CYRUS_IMAPD_SESSION_STATS}|%{CYRUS_IMAPD_SHUTDOWN}|%{CYRUS_IMAPD_SKIPLIST}|%{CYRUS_IMAPD_SSLTLS}|%{CYRUS_IMAPD_UNUSED_CACHE}|%{CYRUS_IMAPD_USAGE}) +CYRUS_IMAPD_IMAPS_MESSAGES (?:%{CYRUS_IMAPD_IMAP_MESSAGES}) +CYRUS_IMAPD_LMTPD_MESSAGES (?:%{CYRUS_DUMMY_FAIL}) +CYRUS_IMAPD_LMTPUNIX_MESSAGES (?:%{CYRUS_IMAPD_AUDITLOG}|%{CYRUS_IMAPD_DUPELIM}|%{CYRUS_IMAPD_LMTPUNIX_DELIVERY}|%{CYRUS_IMAPD_LMTPUNIX_LONGLOCK}|%{CYRUS_IMAPD_LMTPUNIX_SIEVE}|%{CYRUS_IMAPD_MAILBOX_EVENTS}|%{CYRUS_IMAPD_SESSION_STATS}|%{CYRUS_IMAPD_SKIPLIST}|%{CYRUS_IMAPD_USAGE}) +CYRUS_IMAPD_MASTER_MESSAGES (?:%{CYRUS_IMAPD_MASTER_ERRORS}|%{CYRUS_IMAPD_MASTER_STATUS}) +CYRUS_IMAPD_MUPDATE_MESSAGES (?:%{CYRUS_DUMMY_FAIL}) +CYRUS_IMAPD_NOTIFYD_MESSAGES (?:%{CYRUS_IMAPD_EVENT}) +CYRUS_IMAPD_POP3_MESSAGES (?:%{CYRUS_IMAPD_AUDITLOG}|%{CYRUS_IMAPD_DHPARAMS}|%{CYRUS_IMAPD_EXPUNGE}|%{CYRUS_IMAPD_LOGIN}|%{CYRUS_IMAPD_POP_COUNTS}|%{CYRUS_IMAPD_POP_FAILED}|%{CYRUS_IMAPD_SSLTLS}) +CYRUS_IMAPD_POP3S_MESSAGES (?:%{CYRUS_IMAPD_POP3_MESSAGES}) +CYRUS_IMAPD_SIEVE_MESSAGES (?:%{CYRUS_IMAPD_LOGIN}|%{CYRUS_IMAPD_SHUTDOWN}|%{CYRUS_IMAPD_SIEVE_DISCONNECT}|%{CYRUS_IMAPD_SSLTLS}) +CYRUS_IMAPD_SQUATTER_MESSAGES (?:%{CYRUS_IMAPD_MAILBOX_EVENT}|%{CYRUS_SQUATTER}) +CYRUS_IMAPD_TLS_PRUNE_MESSAGES (?:%{CYRUS_TLSPRUNE}) diff --git a/docker/logstash/patterns/postfix b/docker/logstash/patterns/postfix new file mode 100644 index 00000000..03a15be4 --- /dev/null +++ b/docker/logstash/patterns/postfix @@ -0,0 +1,77 @@ +# Postfix stuff based on https://gist.github.com/jbrownsc/4694374: + +PERMERROR 5[0-9]{2} +MESSAGELEVEL reject|warning|error|fatal|panic +POSTFIX_ACTION discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn + +# Postfix +# Queue IDs are hexadecimal, or magic NOQUEUE +QUEUEID (?:[A-F0-9]+|NOQUEUE) + +RELAY (?:%{RELAY_IP}|%{RELAY_SOCKET}) +RELAY_IP (?:%{HOSTNAME:relay_host})?\[(?:%{IP:relay_ip})?\](?::%{POSINT})? +RELAY_SOCKET (?:%{HOSTNAME:relay_host})?\[(?:%{PATH:relay_socket})?\] + +POSREAL [0-9]+(.[0-9]+)? +DELAYS (%{POSREAL}[/]*)+ +DSN %{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT} +STATUS sent|deferred|bounced|expired + +# Permanent errors +PERMERROR 5[0-9]{2} +# Temporary problems +TEMPERROR 4[0-9]{2} + +POSTFIX_ACTION discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn +INVALID_CERT_REASONS (?:certificate has expired|self-signed certificate|untrusted issuer %{DATA:cert_issuer}) + +# These occur on outgoing connections, as in postfix/smtp, but also postfix/lmtp +POSTFIX_4XX (%{POSTFIX_4XX1}|%{POSTFIX_4XX2}) +POSTFIX_4XX1 %{QUEUEID:local_queueid}: host %{RELAY} said: %{GREEDYDATA:reason} +POSTFIX_4XX2 %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=%{RELAY},(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \(host (?:%{HOSTNAME})?\[(?:%{IP}|%{PATH})?\](?::%{POSINT})? said: %{TEMPERROR:responsecode} %{DATA:smtp_response} \(in reply to %{DATA:command} command\)\) +POSTFIX_4XXd NOQUEUE: reject: %{DATA:command} from %{CLIENT}: (?:4[0-9]{2}:error_code}) \d\.\d\.\d <%{EMAIL_ADDRESS:from}>: %{DATA:reason}; from=<%{EMAIL_ADDRESS}> to=%{EMAIL_ADDRESS:to}> proto=%{WORD} helo=<%{DATA:helo}> +POSTFIX_5XX %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=%{RELAY},(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \(host %{RELAY} said: %{PERMERROR:responsecode} %{DATA:smtp_response} \(in reply to %{DATA:command} command\)\) +POSTFIX_CERTINVALID certificate verification failed for %{RELAY}: %{INVALID_CERT_REASONS} +POSTFIX_CONNECT_FROM connect from %{CLIENT} +POSTFIX_CONNECT_TO connect to %{RELAY}: %{GREEDYDATA:reason} +POSTFIX_HOST_STARTTLS Host offered STARTTLS: \[%{IP_OR_HOST}\] +POSTFIX_LOST_CONNECTION %{QUEUEID:local_queueid}: lost connection with %{RELAY} while %{GREEDYDATA:reason} +# Postfix informational message about CISCO PIX SMTP relay "fixup" protocol +POSTFIX_PIX %{QUEUEID:local_queueid}: enabling PIX workarounds: disable_esmtp delay_dotcrlf for %{RELAY} +POSTFIX_REFUSAL %{QUEUEID:local_queueid}: host %{RELAY} refused to talk to me: %{GREEDYDATA:reason} +POSTFIX_RELAY %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=(?:%{RELAY}|local|none),(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \((?:.* SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>\))?(?:.* (forwarded|queued) as %{QUEUEID:remote_queueid}\))?(?:%{DATA:reason}\))? +POSTFIX_TIMEOUT %{QUEUEID:local_queueid}: conversation with %{RELAY} timed out while %{GREEDYDATA:reason} +POSTFIX_TLS_HANDSHAKE %{QUEUEID:local_queueid}: Cannot start TLS: handshake failure + +# Use DATA:to, because people mistype address delimiters, and programmers sometimes... well. +POSTFIX_ACTIONS %{QUEUEID:local_queueid}: %{POSTFIX_ACTION}: %{DATA:command} from %{CLIENT}: %{GREEDYDATA:reason}; (from=<(?:%{EMAIL_ADDRESS:from})?> (to=<%{DATA:to}> )?)?proto=%{DATA:proto} helo=<%{DATA}> +POSTFIX_AUTH_CLIENT (?:(%{QUEUEID:local_queueid}: client=%{CLIENT}(, sasl_method=%{DATA}, sasl_username=%{EMAIL_ADDRESS:client_auth})?)|warning: %{CLIENT}: SASL PLAIN authentication failed: authentication failure) +POSTFIX_CONNECTS (lost connection after %{DATA}|(?:dis)?connect) from %{CLIENT} +POSTFIX_SSL_ACCEPT_ERROR SSL_(?:accept|connect) error (?:from %{CLIENT_SRC}|to %{CLIENT_DEST}): %{DATA} +POSTFIX_TIMEOUTS timeout after %{DATA:command} from %{CLIENT} + +POSTFIX_CLEANUP %{QUEUEID:local_queueid}: (?:(?:resent-)?message-id=(<%{DATA:message_id}>)?|%{POSTFIX_ACTION}: %{DATA:header_replace} from %{CLIENT}; from=<(%{EMAIL_ADDRESS:from})?> to=<%{EMAIL_ADDRESS:to}> proto=%{DATA} helo=<%{DATA}>: %{GREEDYDATA:header_replacement}) +POSTFIX_DSN_NDN %{QUEUEID:local_queueid}: sender (delivery status|non\-delivery) notification: %{QUEUEID:remote_queueid} +POSTFIX_PICKUP %{QUEUEID:local_queueid}: uid=\d+ from=<%{DATA:from}> +POSTFIX_QMGR %{QUEUEID:local_queueid}: (?:removed|from=<(?:%{EMAIL_ADDRESS:from})?>(?:, size=%{POSINT:size}, nrcpt=%{POSINT:nrcpt} \(%{GREEDYDATA:queuestatus}\))?) +POSTFIX_TOO_MANY_ERRORS too many errors after %{DATA:command} from %{CLIENT} +POSTFIX_IMPROPER_COMMAND_PIPELINING improper command pipelining after %{DATA:command} from %{CLIENT}%{DATA} +POSTFIX_STATS statistics: %{GREEDYDATA} +POSTFIX_WARNING warning: %{GREEDYDATA} + +POSTFIX_ANVIL_MESSAGES (?:%{POSTFIX_STATS}) +POSTFIX_BOUNCE_MESSAGES (?:%{POSTFIX_DSN_NDN}) +POSTFIX_CLEANUP_MESSAGES (?:%{POSTFIX_CLEANUP}) +POSTFIX_ERROR_MESSAGES (?:%{POSTFIX_RELAY}) +POSTFIX_LMTP_MESSAGES (?:%{POSTFIX_RELAY}) +POSTFIX_LOCAL_MESSAGES (?:%{POSTFIX_RELAY}|%{POSTFIX_WARNING}) +POSTFIX_MASTER_MESSAGES (?:%{POSTFIX_WARNING}|terminating on signal \d+|daemon started -- version %{DATA}, configuration %{PATH}) +POSTFIX_PICKUP_MESSAGES (?:%{POSTFIX_PICKUP}) +POSTFIX_QMGR_MESSAGES (?:%{POSTFIX_QMGR}|%{POSTFIX_RELAY}|%{POSTFIX_WARNING}) +POSTFIX_SCACHE_MESSAGES (?:%{POSTFIX_STATS}) +POSTFIX_SCRIPT_MESSAGES (?:starting|stopping|waiting for) the Postfix mail system(%{DATA})? +POSTFIX_SMTP_MESSAGES (?:%{POSTFIX_4XX}|%{POSTFIX_5XX}|%{POSTFIX_CERTINVALID}|%{POSTFIX_CONNECT_FROM}|%{POSTFIX_CONNECT_TO}|%{POSTFIX_HOST_STARTTLS}|%{POSTFIX_LOST_CONNECTION}|%{POSTFIX_PIX}|%{POSTFIX_REFUSAL}|%{POSTFIX_RELAY}|%{POSTFIX_TIMEOUT}|%{POSTFIX_TLS_HANDSHAKE}) +POSTFIX_SMTPD_MESSAGES (?:%{POSTFIX_4XXd}|%{POSTFIX_ACTIONS}|%{POSTFIX_AUTH_CLIENT}|%{POSTFIX_CONNECTS}|%{POSTFIX_SSL_ACCEPT_ERROR}|%{POSTFIX_TIMEOUTS}|%{POSTFIX_TOO_MANY_ERRORS}|%{POSTFIX_WARNING}|%{POSTFIX_IMPROPER_COMMAND_PIPELINING}) +POSTFIX_SPAWN_MESSAGES (?:fatal|warning): %{GREEDYDATA:reason} +POSTFIX_TRIVIAL_REWRITE_MESSAGES (?:%{POSTFIX_WARNING}) + diff --git a/docker/logstash/patterns/syslog b/docker/logstash/patterns/syslog new file mode 100644 index 00000000..b5983770 --- /dev/null +++ b/docker/logstash/patterns/syslog @@ -0,0 +1,6 @@ +PROG (?:[\w._/%-]+) + + +SYSLOG_PROG %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])? +SYSLOG_BASE %{TIMESTAMP_ISO8601:syslog_timestamp} %{IP_OR_HOST:syslog_hostname} %{SYSLOG_PROG}: %{GREEDYDATA:syslog_message} + diff --git a/docker/logstash/pipeline/10_input_filesystem_maillog.conf b/docker/logstash/pipeline/10_input_filesystem_maillog.conf new file mode 100644 index 00000000..3d9387a0 --- /dev/null +++ b/docker/logstash/pipeline/10_input_filesystem_maillog.conf @@ -0,0 +1,9 @@ +input { + file { + exclude => [ "*.gz", "*.orig" ] + path => "/data/logs/maillog-*" + sincedb_path => "/usr/share/logstash/.sincedb-maillog" + start_position => beginning + type => "maillog" + } +} diff --git a/docker/logstash/pipeline/11_input_elasticsearch_unparsed.conf b/docker/logstash/pipeline/11_input_elasticsearch_unparsed.conf new file mode 100644 index 00000000..8cf4768e --- /dev/null +++ b/docker/logstash/pipeline/11_input_elasticsearch_unparsed.conf @@ -0,0 +1,7 @@ +input { + elasticsearch { + index => "logstash-*" + query => '{"query": {"match": {"tags":"_unparsed"}}}' + tags => [ "elasticsearch" ] + } +} diff --git a/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf b/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf new file mode 100644 index 00000000..84957fbb --- /dev/null +++ b/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf @@ -0,0 +1,7 @@ +input { + elasticsearch { + index => "logstash-*" + query => '{"query": {"match": {"tags":"_grokparsefailure"}}}' + tags => [ "elasticsearch" ] + } +} diff --git a/docker/logstash/pipeline/13_input_elasticsearch_groktimeout.conf b/docker/logstash/pipeline/13_input_elasticsearch_groktimeout.conf new file mode 100644 index 00000000..24f78d48 --- /dev/null +++ b/docker/logstash/pipeline/13_input_elasticsearch_groktimeout.conf @@ -0,0 +1,8 @@ +# These entries have shown grok timeouts, so try them again +input { + elasticsearch { + index => "logstash-*" + query => '{"query": {"match": {"tags": "_groktimeout"}}}' + tags => [ "elasticsearch" ] + } +} diff --git a/docker/logstash/pipeline/13_input_redis.conf b/docker/logstash/pipeline/13_input_redis.conf new file mode 100644 index 00000000..f5106090 --- /dev/null +++ b/docker/logstash/pipeline/13_input_redis.conf @@ -0,0 +1,7 @@ +# Intended for Redis input from the kolab system +#input { +# redis { +# host => "redis" +# port => 6379 +# } +#} diff --git a/docker/logstash/pipeline/20_filter_metrics.conf b/docker/logstash/pipeline/20_filter_metrics.conf new file mode 100644 index 00000000..75bfc58b --- /dev/null +++ b/docker/logstash/pipeline/20_filter_metrics.conf @@ -0,0 +1,6 @@ +filter { + metrics { + meter => "events" + add_tag => "metric" + } +} diff --git a/docker/logstash/pipeline/21_filter_elasticsearch_input.conf b/docker/logstash/pipeline/21_filter_elasticsearch_input.conf new file mode 100644 index 00000000..0d5cd909 --- /dev/null +++ b/docker/logstash/pipeline/21_filter_elasticsearch_input.conf @@ -0,0 +1,13 @@ +filter { + if "elasticsearch" in [tags] { + if [orig_message] { + prune { + whitelist_names => [ "^orig_message$" ] + } + + mutate { + rename => [ "orig_message", "message" ] + } + } + } +} diff --git a/docker/logstash/pipeline/22_filter_maillog.conf b/docker/logstash/pipeline/22_filter_maillog.conf new file mode 100644 index 00000000..6af7197c --- /dev/null +++ b/docker/logstash/pipeline/22_filter_maillog.conf @@ -0,0 +1,276 @@ +filter { + if [type] == "maillog" { + # fingerprint the original message + fingerprint { + source => [ "message" ] + target => "[@metadata][fingerprint]" + method => "SHA512" + key => "logstash" + } + + mutate { + add_field => { "orig_message" => "%{message}" } + } + + # The kolab syslog base matches the RSYSLOG_TraditionalFormat + grok { + match => { + "message" => "%{SYSLOG_BASE}" + } + patterns_dir => "/usr/share/logstash/patterns/" + } + + # Of which the date is in the ISO8601 format. Note that using + # 'date' here sets the Logstash/Elasticsearch timestamp. + date { + match => [ "syslog_timestamp", "ISO8601" ] + } + + if "_grokparsefailure" not in [tags] { + mutate { + replace => [ "host", "%{syslog_hostname}" ] + replace => [ "message", "%{syslog_message}" ] + } + + mutate { + remove_field => [ + "path", + "syslog_hostname", + "syslog_message", + "syslog_timestamp" + ] + } + } + + if [syslog_program] == "ctl_cyrusdb" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "ctl_mboxlist" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "cyr_expire" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "imap" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + + # Specialty MyKolab -> Kolab Now item + } else if [syslog_program] == "imapkn" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/imap" ] + } + + } else if [syslog_program] == "imaps" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + + # Specialty MyKolab -> Kolab Now item + } else if [syslog_program] == "imapskn" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/imaps" ] + } + + } else if [syslog_program] == "lmtpunix" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "pop3" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "pop3s" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "squatter" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } else if [syslog_program] == "tls_prune" { + mutate { + replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] + } + } + + if [syslog_program] =~ /^cyrus-imapd\// { + if [syslog_program] == "cyrus-imapd/ctl_cyrusdb" { + grok { + match => [ "message", "%{CYRUS_IMAPD_CTL_CYRUSDB_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/cyr_expire" { + grok { + match => [ "message", "%{CYRUS_IMAPD_CYR_EXPIRE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/imap" { + grok { + match => [ "message", "%{CYRUS_IMAPD_IMAP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/imaps" { + grok { + match => [ "message", "%{CYRUS_IMAPD_IMAPS_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/lmtpd" { + grok { + match => [ "message", "%{CYRUS_IMAPD_LMTPD_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/lmtpunix" { + grok { + match => [ "message", "%{CYRUS_IMAPD_LMTPUNIX_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/master" { + grok { + match => [ "message", "%{CYRUS_IMAPD_MASTER_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/mupdate" { + grok { + match => [ "message", "%{CYRUS_IMAPD_MUPDATE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/notifyd" { + grok { + match => [ "message", "%{CYRUS_IMAPD_NOTIFYD_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/pop3" { + grok { + match => [ "message", "%{CYRUS_IMAPD_POP3_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/pop3s" { + grok { + match => [ "message", "%{CYRUS_IMAPD_POP3S_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/timsieved" { + grok { + match => [ "message", "%{CYRUS_IMAPD_SIEVE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/squatter" { + grok { + match => [ "message", "%{CYRUS_IMAPD_SQUATTER_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "cyrus-imapd/tls_prune" { + grok { + match => [ "message", "%{CYRUS_IMAPD_TLS_PRUNE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else { + mutate { + add_tag => [ "_unparsed" ] + } + } + } + + else if [syslog_program] =~ /^postfix\// { + if [syslog_program] == "postfix/anvil" { + grok { + match => [ "message", "%{POSTFIX_ANVIL_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/bounce" { + grok { + match => [ "message", "%{POSTFIX_BOUNCE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/cleanup" { + grok { + match => [ "message", "%{POSTFIX_CLEANUP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/error" { + grok { + match => [ "message", "%{POSTFIX_ERROR_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/lmtp" { + grok { + match => [ "message", "%{POSTFIX_LMTP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/local" { + grok { + match => [ "message", "%{POSTFIX_LOCAL_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/master" { + grok { + match => [ "message", "%{POSTFIX_MASTER_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/pickup" { + grok { + match => [ "message", "%{POSTFIX_PICKUP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/postfix-script" { + grok { + match => [ "message", "%{POSTFIX_SCRIPT_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/qmgr" { + grok { + match => [ "message", "%{POSTFIX_QMGR_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/scache" { + grok { + match => [ "message", "%{POSTFIX_SCACHE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/smtp" { + grok { + match => [ "message", "%{POSTFIX_SMTP_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/smtpd" { + grok { + match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/smtps/smtpd" { + grok { + match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/submission/smtpd" { + grok { + match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/spawn" { + grok { + match => [ "message", "%{POSTFIX_SPAWN_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else if [syslog_program] == "postfix/trivial-rewrite" { + grok { + match => [ "message", "%{POSTFIX_TRIVIAL_REWRITE_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + } else { + mutate { + add_tag => [ "_unparsed" ] + } + } + } else { + mutate { + add_tag => [ "_unparsed" ] + } + } + } +} diff --git a/docker/logstash/pipeline/30_network.conf b/docker/logstash/pipeline/30_network.conf new file mode 100644 index 00000000..34508775 --- /dev/null +++ b/docker/logstash/pipeline/30_network.conf @@ -0,0 +1,158 @@ +filter { + if [dest_ip] { + cidr { + address => [ "%{dest_ip}" ] + network => [ + "0::/16", # unknown + "fc00::/7", # local private network range + "fe80::/10", # link-local + "ff00::/16", # reserved + "ff01::/16", # interface-local + "ff02::/16", # link-local + "ff03::/16", # IPv4 local scope + "ff04::/16", # admin-local + "ff05::/16", # site-local + "ff06::/16", # reserved + "ff07::/16", # reserved + "ff08::/16", # organization-local + "ff09::/16", # reserved + "ff0a::/16", # reserved + "ff0b::/16", # reserved + "ff0c::/16", # reserved + "ff0d::/16", # reserved + "ff0e::/16", # global scope + "ff0f::/16" # reserved + ] + add_tag => [ "domestic_dest" ] + } + + cidr { + address => [ "%{dest_ip}" ] + network => [ + "0.0.0.0/32", # unknown + "224.0.0.0/24", # local subnetwork + "224.0.1.0/24", # internetwork control + "224.0.2.0/24", # ad-hoc block 1 + "224.3.0.0/16", # ad-hoc block 2 + "224.4.0.0/16", # ad-hoc block 2 + "232.0.0.0/8", # source-specific multicast + "233.252.0.0/6", # ad-hoc block 3 + "234.0.0.0/8", # unicast-prefix based + "239.0.0.0/8", # administratively scoped + "255.255.255.255/32" # broadcast + ] + add_tag => [ "domestic_dest" ] + } + + cidr { + address => [ "%{src_ip}" ] + network => [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] + add_tag => [ "domestic_dest" ] + } + + cidr { + add_tag => [ "localhost" ] + address => [ "%{dest_ip}" ] + network => [ "127.0.0.1/8" ] + } + } + + if [src_ip] { + cidr { + address => [ "%{src_ip}" ] + network => [ + "0::/16", # unknown + "fc00::/7", # local private network range + "fe80::/10", # link-local + "ff00::/16", # reserved + "ff01::/16", # interface-local + "ff02::/16", # link-local + "ff03::/16", # IPv4 local scope + "ff04::/16", # admin-local + "ff05::/16", # site-local + "ff06::/16", # reserved + "ff07::/16", # reserved + "ff08::/16", # organization-local + "ff09::/16", # reserved + "ff0a::/16", # reserved + "ff0b::/16", # reserved + "ff0c::/16", # reserved + "ff0d::/16", # reserved + "ff0e::/16", # global scope + "ff0f::/16" # reserved + ] + add_tag => [ "domestic_src" ] + } + + cidr { + address => [ "%{src_ip}" ] + network => [ + "0.0.0.0/32", # unknown + "224.0.0.0/24", # local subnetwork + "224.0.1.0/24", # internetwork control + "224.0.2.0/24", # ad-hoc block 1 + "224.3.0.0/16", # ad-hoc block 2 + "224.4.0.0/16", # ad-hoc block 2 + "232.0.0.0/8", # source-specific multicast + "233.252.0.0/6", # ad-hoc block 3 + "234.0.0.0/8", # unicast-prefix based + "239.0.0.0/8", # administratively scoped + "255.255.255.255/32" # broadcast + ] + add_tag => [ "domestic_src" ] + } + + cidr { + address => [ "%{src_ip}" ] + network => [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] + add_tag => [ "domestic_src" ] + } + + cidr { + add_tag => [ "localhost" ] + address => [ "%{src_ip}" ] + network => [ "127.0.0.1/8" ] + } + } + + if [dest_ip] or [src_ip] { + if "localhost" in [tags] { + mutate { + remove_tag => [ "domestic_dest", "domestic_src" ] + } + } else { + if "domestic_dest" in [tags] { + if "domestic_src" in [tags] { + mutate { + add_tag => [ "domestic" ] + remove_tag => [ "domestic_dest", "domestic_src" ] + } + } else { + mutate { + add_tag => [ "inbound" ] + remove_tag => [ "domestic_dest" ] + } + } + } else { + if "domestic_src" in [tags] { + mutate { + add_tag => [ "outbound" ] + remove_tag => [ "domestic_src" ] + } + } else { + mutate { + add_tag => [ "foreign" ] + } + } + } + } + } +} diff --git a/docker/logstash/pipeline/31_geoip.conf b/docker/logstash/pipeline/31_geoip.conf new file mode 100644 index 00000000..79c724d6 --- /dev/null +++ b/docker/logstash/pipeline/31_geoip.conf @@ -0,0 +1,79 @@ +filter { + if "inbound" in [tags] { + geoip { + source => "src_ip" + target => "geo_src" + } + + if "_geoip_lookup_failure" not in [tags] { + mutate { + add_field => { + "location_src" => "%{[geo_src][location][lat]},%{[geo_src][location][lon]}" + } + } + } else { + mutate { + remove_tag => [ "_geoip_lookup_failure" ] + } + } + } + + if "outbound" in [tags] { + geoip { + source => "dest_ip" + target => "geo_dest" + } + + if "_geoip_lookup_failure" not in [tags] { + mutate { + add_field => { + "location_dest" => "%{[geo_dest][location][lat]},%{[geo_dest][location][lon]}" + } + } + } else { + mutate { + remove_tag => [ "_geoip_lookup_failure" ] + } + } + } + + if "foreign" in [tags] { + if [dest_ip] { + geoip { + source => "dest_ip" + target => "geo_dest" + } + + if "_geoip_lookup_failure" not in [tags] { + mutate { + add_field => { + "location_dest" => "%{[geo_dest][location][lat]},%{[geo_dest][location][lon]}" + } + } + } else { + mutate { + remove_tag => [ "_geoip_lookup_failure" ] + } + } + } + + if [src_ip] { + geoip { + source => "src_ip" + target => "geo_src" + } + + if "_geoip_lookup_failure" not in [tags] { + mutate { + add_field => { + "location_src" => "%{[geo_src][location][lat]},%{[geo_src][location][lon]}" + } + } + } else { + mutate { + remove_tag => [ "_geoip_lookup_failure" ] + } + } + } + } +} diff --git a/docker/logstash/pipeline/90_output_elasticsearch_maillog.conf b/docker/logstash/pipeline/90_output_elasticsearch_maillog.conf new file mode 100644 index 00000000..dc785c3c --- /dev/null +++ b/docker/logstash/pipeline/90_output_elasticsearch_maillog.conf @@ -0,0 +1,12 @@ +output { + if [type] == "maillog" { + elasticsearch { + document_id => "%{[@metadata][fingerprint]}" + hosts => [ "127.0.0.1:9200" ] + index => "logstash-1.0.0-%{+YYYY.MM.dd}" + template => "/usr/share/logstash/templates/logstash_template.json" + template_name => "logstash-1.0.0" + template_overwrite => true + } + } +} diff --git a/docker/logstash/pipeline/91_output_stdout_grokparsefailure.conf b/docker/logstash/pipeline/91_output_stdout_grokparsefailure.conf new file mode 100644 index 00000000..0ceb3ac3 --- /dev/null +++ b/docker/logstash/pipeline/91_output_stdout_grokparsefailure.conf @@ -0,0 +1,7 @@ +#output { +# if "_grokparsefailure" in [tags] { +# stdout { +# codec => rubydebug +# } +# } +#} diff --git a/docker/logstash/pipeline/92_output_file_grokparsefailure.conf b/docker/logstash/pipeline/92_output_file_grokparsefailure.conf new file mode 100644 index 00000000..d6411ade --- /dev/null +++ b/docker/logstash/pipeline/92_output_file_grokparsefailure.conf @@ -0,0 +1,15 @@ +#output { +# if "_grokparsefailure" in [tags] { +# file { +# path => "/usr/share/logstash/_grokparsefailures/%{[@metadata][fingerprint]}.log" +# codec => line { +# format => "%{orig_message}" +# } +# } +# +# file { +# path => "/usr/share/logstash/_grokparsefailures/%{[@metadata][fingerprint]}.json" +# codec => json +# } +# } +#} diff --git a/docker/logstash/pipeline/99_output_metrics.conf b/docker/logstash/pipeline/99_output_metrics.conf new file mode 100644 index 00000000..f152a76e --- /dev/null +++ b/docker/logstash/pipeline/99_output_metrics.conf @@ -0,0 +1,9 @@ +output { + if ("metric" in [tags]) { + stdout { + codec => line { + format => "rate: %{[events][rate_1m]}, %{[events][rate_5m]}, %{[events][rate_15m]} count: %{[events][count]}" + } + } + } +} diff --git a/docker/logstash/templates/logstash_template.json b/docker/logstash/templates/logstash_template.json new file mode 100644 index 00000000..932ba0cc --- /dev/null +++ b/docker/logstash/templates/logstash_template.json @@ -0,0 +1,107 @@ +{ + "mappings": { + "dynamic_templates": [ + { + "message_field": { + "mapping": { + "norms": false, + "type": "text" + }, + "match_mapping_type": "string", + "path_match": "message" + } + }, + { + "string_fields": { + "mapping": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "norms": false, + "type": "text" + }, + "match": "*", + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "type": "keyword" + }, + "db_bytes": { + "type": "integer" + }, + "db_path": { + "type": "keyword" + }, + "db_records": { + "type": "integer" + }, + "db_seconds": { + "type": "half_float" + }, + "geoip_dest": { + "dynamic": true, + "properties": { + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + } + } + }, + "geoip_src": { + "dynamic": true, + "properties": { + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + } + } + }, + "location_dest": { + "type": "geo_point" + }, + "location_src": { + "type": "geo_point" + }, + "syslog_pid": { + "type": "integer" + }, + "syslog_program": { + "type": "keyword" + } + } + }, + "order": 0, + "settings": { + "index": { + "number_of_shards": "1", + "refresh_interval": "5s" + } + }, + "template": "logstash-1.0.0-*", + "version": 60001 +}