diff --git a/bin/reinit-logstash.sh b/bin/reinit-logstash.sh index caa2b4df..8a8f50fe 100755 --- a/bin/reinit-logstash.sh +++ b/bin/reinit-logstash.sh @@ -1,19 +1,21 @@ #!/bin/bash docker-compose stop logstash for index in $(curl localhost:9200/_cat/indices 2>/dev/null | awk '{print $3}' | grep ^logstash) do + echo -n "Deleting ${index} .. " curl -X DELETE localhost:9200/${index} >/dev/null 2>&1 + echo "DONE." done find docker/logstash/_grokparsefailures/ -type f ! -name ".gitignore" -delete curl -X DELETE localhost:9200/_template/logstash-1.0.0 >/dev/null 2>&1 curl -X PUT -H 'Content-Type: application/json' \ -d "$(cat docker/logstash/templates/logstash_template.json)" \ localhost:9200/_template/logstash-1.0.0 >/dev/null 2>&1 docker-compose up -d --force-recreate logstash docker-compose logs -f logstash diff --git a/docker-compose.yml b/docker-compose.yml index 8f0a8476..8ab987a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,103 +1,103 @@ version: '2.1' services: elasticsearch: container_name: elasticsearch environment: - cluster.routing.allocation.disk.threshold_enabled=false - discovery.type=single-node hostname: elasticsearch image: elasticsearch:7.3.2 network_mode: host volumes: - /data/elasticsearch/kolab/elasticsearch/:/usr/share/elasticsearch/data/ kibana: container_name: kibana extra_hosts: - "elasticsearch:127.0.0.1" hostname: kibana image: kibana:7.3.2 network_mode: host kolab: build: context: ./docker/kolab/ container_name: kolab depends_on: mariadb: condition: service_healthy extra_hosts: - "kolab.mgmt.com:127.0.0.1" healthcheck: interval: 10s test: test -f /tmp/kolab-init.done timeout: 5s retries: 30 hostname: kolab.mgmt.com network_mode: host tmpfs: - /run - /tmp - /var/run - /var/tmp tty: true volumes: - ./docker/certs/ca.cert:/etc/pki/tls/certs/ca.cert:ro - ./docker/certs/ca.cert:/etc/pki/ca-trust/source/anchors/ca.cert:ro - ./docker/certs/kolab.hosted.com.cert:/etc/pki/tls/certs/kolab.hosted.com.cert - ./docker/certs/kolab.hosted.com.key:/etc/pki/tls/certs/kolab.hosted.com.key - ./docker/certs/kolab.mgmt.com.cert:/etc/pki/tls/certs/kolab.mgmt.com.cert - ./docker/certs/kolab.mgmt.com.key:/etc/pki/tls/certs/kolab.mgmt.com.key - ./docker/kolab/utils:/root/utils:ro - /sys/fs/cgroup:/sys/fs/cgroup:ro logstash: container_name: logstash-kolab environment: - config.reload.automatic=true #- pipeline.workers=1 #- pipeline.batch.size=1 - xpack.monitoring.enabled=true extra_hosts: - "elasticsearch:127.0.0.1" - "redis:127.0.0.1" hostname: logstash image: logstash:7.3.2 network_mode: host volumes: - ./docker/logstash/_grokparsefailures/:/usr/share/logstash/_grokparsefailures/ - ./docker/logstash/patterns/:/usr/share/logstash/patterns/:ro - ./docker/logstash/pipeline/:/usr/share/logstash/pipeline/:ro - ./docker/logstash/templates/:/usr/share/logstash/templates/:ro - #- ./docker/logstash/data/:/data/logs/:ro + - ./docker/logstash/data/:/usr/share/logstash/mydata/:ro - /data/logs/:/data/logs/:ro mariadb: container_name: mariadb environment: MYSQL_ROOT_PASSWORD: Welcome2KolabSystems healthcheck: interval: 10s test: test -e /var/run/mysqld/mysqld.sock timeout: 5s retries: 30 image: mariadb network_mode: host redis: container_name: redis-kolab image: redis network_mode: host worker: build: context: ./docker/worker/ container_name: worker depends_on: kolab: condition: service_healthy hostname: worker network_mode: host tmpfs: - /run - /tmp - /var/run - /var/tmp tty: true volumes: - ./src:/home/worker/src.orig:ro - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/docker/logstash/patterns/base b/docker/logstash/patterns/base index edcd8073..4fe8299f 100644 --- a/docker/logstash/patterns/base +++ b/docker/logstash/patterns/base @@ -1,32 +1,36 @@ DATA .*? GREEDYDATA .* # RFC 5322 at http://tools.ietf.org/html/rfc5322#section-3.4 EMAIL_ADDRESS_LOCAL (?:[a-zA-Z0-9!#\$%&'\*\+/=\?\^_`{\|}~\-]+(?:\.[a-zA-Z0-9!#\$%&'\*\+/=\?\^_`{\|}~\-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*") EMAIL_ADDRESS_DOMAIN (?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-zA-Z0-9-]*[a-zA-Z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]) EMAIL_ADDRESS %{EMAIL_ADDRESS_LOCAL}(?:@%{EMAIL_ADDRESS_DOMAIN:domain})? IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? IPV4 (?\d\d){1,2} MONTHNUM (?:0?[1-9]|1[0-2]) MONTHDAY (?:(?:(0|\s)[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) HOUR (?:2[0123]|[01]?[0-9]) MINUTE (?:[0-5][0-9]) SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) POSINT \b(?:[1-9][0-9]*)\b NONNEGINT \b(?:[0-9]+)\b TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? # A client address as listed in Postfix or Cyrus IMAP messages CLIENT ((?:(%{HOSTNAME:client_host}|%{IP_OR_HOST:client_host}|unknown)\s?)?)?\[(?:%{IP:src_ip}|unknown)?\](?::%{POSINT})? CLIENT_DEST ((?:(%{HOSTNAME:client_host}|%{IP_OR_HOST:client_host}|unknown)\s?)?)?\[(?:%{IP:dest_ip}|unknown)?\](?::%{POSINT})? CLIENT_SRC ((?:(%{HOSTNAME:client_host}|%{IP_OR_HOST:client_host}|unknown)\s?)?)?\[(?:%{IP:src_ip}|unknown)?\](?::%{POSINT})? diff --git a/docker/logstash/patterns/cyrus-imapd b/docker/logstash/patterns/cyrus-imapd index 6aa29bbc..ab259ef7 100644 --- a/docker/logstash/patterns/cyrus-imapd +++ b/docker/logstash/patterns/cyrus-imapd @@ -1,94 +1,95 @@ CYRUS_IMAP_SESSIONID (?:%{IP_OR_HOST}|%{DATA})-\d+-\d+-\d+(-\d+)? BADLOGIN_REASONS \[?(SASL(\(-13\):)? )?(authentication failure: checkpass failed|user not found: could not find password|(authentication failure: )?Password verification failed|authentication failure)\]? CYRUS_DUMMY_FAIL fail me CYRUS_IMAPD_AUDIT_ACTIONS (?:append|discard|duplicate|modseq|proxy|redirect|traffic) CYRUS_IMAPD_SIEVE_ACTIONS (discarded|redirected|rejected) CYRUS_IMAPD_AUDITLOG auditlog: %{CYRUS_IMAPD_AUDIT_ACTIONS} (?:%{EMAIL_ADDRESS:client_auth} )?(?:sessionid=<%{CYRUS_IMAP_SESSIONID:session_id}> )?(?:action=<%{CYRUS_IMAPD_AUDIT_ACTIONS}> )?(?:bytes_in=<%{BASE10NUM:bytes_in}> bytes_out=<%{BASE10NUM:bytes_out}>)?(?:message-id=<%{DATA:message_id}>(?:-%{EMAIL_ADDRESS:client_auth} user=<%{DATA}> date=<%{DATA}>)?(?:mailbox=<%{DATA:mailbox}> uniqueid=<%{DATA:uniqueid}>(?: highestmodseq=<%{BASE10NUM:modseq}>|uid=<%{BASE10NUM}> guid=<%{DATA}> cid=<%{DATA}>)?)?)? CYRUS_IMAPD_CHECKING_DATABASES (?:checkpointing cyrus databases|(?:done )?(?:checkpointing|recovering) cyrus databases) CYRUS_IMAPD_CLIENTID client id: (?:(?:"name" "%{DATA:client_name}" "version" "%{DATA:client_version}") )?%{GREEDYDATA:client_extra} CYRUS_IMAPD_DELPRUNE Removed \d+ deleted mailboxes CYRUS_IMAPD_DUPELIM dupelim: eliminated duplicate message to %{DATA:mailbox} id <%{DATA:message_id}>(?:-%{EMAIL_ADDRESS:client_auth})? date %{DATESTAMP_RFC2822} (\(%{DATA}\) )?\((?:delivery|redirect)\) CYRUS_IMAPD_DUPPRUNE duplicate_prune: %{GREEDYDATA} CYRUS_IMAPD_EVENT EVENT, , , , %{GREEDYDATA:event_json} CYRUS_IMAPD_EXPUNGE (Expired \d+ and )?(E|e)xpunged \d+ (out of \d+ )?messages from (?:%{DATA:mailbox}|\d+ mailboxes) CYRUS_IMAPD_INDEX_UPGRADE Index upgrade: %{DATA:mailbox} \(\d+ -> \d+\) CYRUS_IMAPD_LMTPUNIX_DELIVERY Delivered:\s+(<)?%{DATA:message_id(>)? to mailbox: %{GREEDYDATA:mailbox} CYRUS_IMAPD_LONGLOCK (mailbox|skiplist): longlock %{DATA} for [0-9]+\.[0-9]+ seconds CYRUS_IMAPD_LMTPUNIX_SIEVE (?:%{CYRUS_IMAPD_SIEVE_ACTION}|%{CYRUS_IMAPD_SIEVE_RUNTIME_ERROR}) # Note that sometimes a reverse DNS record is simply a '.', because it is virtually unrestricted. -CYRUS_IMAPD_LOGIN (?:bad)?login: (?:%{CLIENT}|\.) (?:%{WORD:mech} )?(?:(\(<)?%{EMAIL_ADDRESS:client_auth}(>\))? )?(?:%{WORD:mech}(\+TLS)? )?(?:(User logged in( SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>)?)|(%{BADLOGIN_REASONS:reason})?) +#CYRUS_IMAPD_LOGIN (?:bad)?login: (?:%{CLIENT}|\.) (?:%{WORD:mech} )?(?:(\(<)?%{EMAIL_ADDRESS:client_auth}(>\))? )?(?:%{WORD:mech}(\+TLS)? )?(?:(User logged in( SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>)?)|(%{BADLOGIN_REASONS:reason})?) +CYRUS_IMAPD_LOGIN (?:bad)?login: (?:%{CLIENT}|\.) (?:(\(<)?%{EMAIL_ADDRESS:client_auth}(>\)|unknown)? )?(?:(%{WORD:mech}\+)(TLS)? )?(?:(User logged in( SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>)?)|(%{BADLOGIN_REASONS:reason})?) CYRUS_IMAPD_MAILBOX_DELETE Deleted mailbox %{DATA:mailbox} CYRUS_IMAPD_MAILBOX_OPEN open: user %{EMAIL_ADDRESS:client_auth} opened %{DATA:mailbox} CYRUS_IMAPD_MAILBOX_EVENT (indexing|Deleted|Repacking) mailbox %{DATA:mailbox}(?:(\.\.\. | version \d+))? CYRUS_IMAPD_MAILBOX_RENAME Rename: %{DATA:mailbox} -> %{DATA:mailbox} CYRUS_IMAPD_MASTER_STATUS (?:service %{WORD} pid \d+ in %{WORD} state: terminated abnormally|(?:process type:SERVICE name:%{NOTSPACE} path:%{PATH} age:%{POSREAL}s pid:%{POSINT} )?(?:exiting|exited)(?:, status %{BASE10NUM})?) CYRUS_IMAPD_NETWORK_ERROR (Fatal error: Lost connection to selected backend|(?:Connection (?:timed out|reset by peer)|Error decompressing data|idle for too long|Invalid argument|No route to host), closing connection) CYRUS_IMAPD_NONOTIFY unable to sendto\(\) notify socket: No such file or directory CYRUS_IMAPD_POP_COUNTS counts: retr=<\d+> top=<\d+> dele=<\d+> CYRUS_IMAPD_POP_FAILED pop3(s)? failed: %{CLIENT} CYRUS_IMAPD_PROXY proxy %{EMAIL_ADDRESS:client_auth} session-id=<%{CYRUS_IMAP_SESSIONID:session_id}> remote=<%{CYRUS_IMAP_SESSIONID:session_id}> CYRUS_IMAPD_SAME_MSG %{DATA:mailbox}: same message appears twice %{INT} %{INT} CYRUS_IMAPD_SESSION_STATS (?:IMAP|LMTP) session stats : I/O read : %{BASE10NUM:bytes_read} bytes : I/O write : %{BASE10NUM:bytes_write} bytes CYRUS_IMAPD_SETRLIMIT (?:setrlimit: Unable to set file descriptors limit to -1: Operation not permitted|retrying with \d+ \(current max\)) CYRUS_IMAPD_SETSOCKETOPT unable to setsocketopt\(IP_TOS\)(?: service %{DATA})?: Operation not supported CYRUS_IMAPD_SHUTDOWN (All children have exited, closing down|attempting clean shutdown on signal|graceful shutdown) CYRUS_IMAPD_SIEVE_ACTION sieve %{CYRUS_IMAPD_SIEVE_ACTIONS}: <%{DATA:message_id}>(?: to: %{EMAIL_ADDRESS:to})? CYRUS_IMAPD_SIEVE_DISCONNECT Lost connection to client -- exiting CYRUS_IMAPD_SIEVE_RUNTIME_ERROR sieve runtime error for %{EMAIL_ADDRESS:client_auth} id <%{DATA:message_id}>: Fileinto: Mailbox does not exist CYRUS_IMAPD_SKIPLIST (?:(skiplist|twoskip)): (checkpointed %{PATH} \(%{INT} records, (%{INT} => )?%{INT} bytes\) in %{POSREAL} sec(ond(s)?)?|clean shutdown file missing, updating recovery stamp) CYRUS_IMAPD_UNUSED_CACHE Removed unused cache file %{PATH} CYRUS_IMAPD_USAGE USAGE %{EMAIL_ADDRESS:client_auth} user: %{POSREAL} sys: %{POSREAL} CYRUS_IMAPD_MAILBOX_EVENTS (%{CYRUS_IMAPD_MAILBOX_DELETE}|%{CYRUS_IMAPD_MAILBOX_OPEN}|%{CYRUS_IMAPD_MAILBOX_EVENT}|%{CYRUS_IMAPD_MAILBOX_RENAME}) CYRUS_IMAPD_ERRORS (?:%{CYRUS_IMAPD_IOERRORS}|%{CYRUS_IMAPD_NETWORK_ERROR}|%{CYRUS_IMAPD_NONOTIFY}) CYRUS_IMAPD_MASTER_ERRORS (?:%{CYRUS_IMAPD_NONOTIFY}|%{CYRUS_IMAPD_SETRLIMIT}|%{CYRUS_IMAPD_SETSOCKETOPT}|%{CYRUS_IMAPD_SHUTDOWN}) CYRUS_IMAPD_CERT_EXPIRED cert has expired CYRUS_IMAPD_DHPARAMS ^inittls: Loading (hard-coded DH parameters|DH parameters from file)$ CYRUS_IMAPD_SCARF scarfing mailbox list from master mupdate server CYRUS_IMAPD_MUPDATE_SUCCESS successful mupdate connection to %{IP_OR_HOST} CYRUS_IMAPD_MUPDATE_SYNC synchronizing mailbox list with master mupdate server CYRUS_IMAPD_MUPDATE_SYNCOK mailbox list synchronization complete CYRUS_IMAPD_MUPDATE_UNREADY unready for connections CYRUS_CHECKPOINT checkpointing cyrus databases # <22>May 24 13:30:08 kolab11 cyrus-imapd/imap[19410]: client id: "name" "Roundcube" "version" "1.0.0" "php" "5.3.27" "os" "Linux" "command" "/?_task=mail&_refresh=1&_mbox=INBOX&_action=list&_remote=1&_unlock=loading1400931009534&_=1400924844797" CYRUS_CONNECT accepted connection CYRUS_DBERROR DBERROR: %{GREEDYDATA} CYRUS_EXEC (?:about to exec %{DATA}|executed) CYRUS_EXIT process %{BASE10NUM:pid} exited, status %{BASE10NUM:pid_exit} CYRUS_EXPIRE unable to open mailbox %{DATA:mailbox} CYRUS_IDLED error sending to idled: %{BASE10NUM} CYRUS_IOERROR_INDEX IOERROR: (?:opening index %{DATA:mailbox}: %{DATA:reason}|%{GREEDYDATA:reason}) CYRUS_IOERROR_MESSAGE IOERROR: reading message: %{DATA:reason} CYRUS_QRESYNC inefficient qresync \(%{POSINT} > %{POSINT}\) %{DATA:mailbox} CYRUS_SQUATTER (?:(done )?indexing mailboxes|error opening %{DATA:mailbox}: %{DATA:reason}|timezone shift for squatter - altering schedule by %{BASE10NUM} seconds|SQUAT returned \d+ messages) CYRUS_IMAPD_IOERRORS (%{CYRUS_IOERROR_INDEX}|%{CYRUS_IOERROR_MESSAGE}) CYRUS_SSLACCEPT SSL_accept\(\) %{WORD} -> %{WORD} CYRUS_STARTTLS starttls: %{DATA:tls_version} with cipher %{DATA:tls_cipher} \(\d+/\d+ bits new\) no authentication CYRUS_TLSERROR (?:Fatal error: (?:Lost connection to input stream|tls_start_servertls\(\) failed)|verify error:num=(?:18:self signed certificate|10:certificate has expired)) CYRUS_TLSNEG (?:(\[pop3d\] )?(imaps TLS|STARTTLS)) (negotiation )?failed: %{CLIENT} CYRUS_TLSPRUNE tls_prune: purged \d+ out of \d+ entries CYRUS_IMAPD_SSLTLS (%{CYRUS_SSLACCEPT}|%{CYRUS_STARTTLS}|%{CYRUS_TLSERROR}|%{CYRUS_TLSNEG}|%{CYRUS_TLSPRUNE}) CYRUS_IMAPD_CTL_CYRUSDB_MESSAGES (?:%{CYRUS_IMAPD_CHECKING_DATABASES}|%{CYRUS_IMAPD_SKIPLIST}) CYRUS_IMAPD_CTL_MBOXLIST_MESSAGES (?:%{CYRUS_DBERROR}) CYRUS_IMAPD_CYR_EXPIRE_MESSAGES (?:%{CYRUS_IMAPD_DELPRUNE}|%{CYRUS_IMAPD_DUPPRUNE}|%{CYRUS_IMAPD_EXPUNGE}|%{CYRUS_IMAPD_MAILBOX_EVENT}|%{CYRUS_IMAPD_LONGLOCK}) CYRUS_IMAPD_CYR_INFO_MESSAGES (?:%{CYRUS_DBERROR}) CYRUS_IMAPD_IMAP_MESSAGES (?:%{BADLOGIN_REASONS}|%{CYRUS_IMAPD_AUDITLOG}|%{CYRUS_IMAPD_CERT_EXPIRED}|%{CYRUS_IMAPD_CLIENTID}|%{CYRUS_IMAPD_DHPARAMS}|%{CYRUS_IMAPD_EXPUNGE}|%{CYRUS_IMAPD_ERRORS}|%{CYRUS_IMAPD_INDEX_UPGRADE}|%{CYRUS_IMAPD_LOGIN}|%{CYRUS_IMAPD_LONGLOCK}|%{CYRUS_IMAPD_MAILBOX_EVENTS}|%{CYRUS_IMAPD_PROXY}|%{CYRUS_IMAPD_SAME_MSG}|%{CYRUS_IMAPD_SESSION_STATS}|%{CYRUS_IMAPD_SHUTDOWN}|%{CYRUS_IMAPD_SKIPLIST}|%{CYRUS_IMAPD_SSLTLS}|%{CYRUS_IMAPD_UNUSED_CACHE}|%{CYRUS_IMAPD_USAGE}) CYRUS_IMAPD_IMAPS_MESSAGES (?:%{CYRUS_IMAPD_IMAP_MESSAGES}) CYRUS_IMAPD_LMTPD_MESSAGES (?:%{CYRUS_DUMMY_FAIL}) CYRUS_IMAPD_LMTPUNIX_MESSAGES (?:%{CYRUS_IMAPD_AUDITLOG}|%{CYRUS_IMAPD_DUPELIM}|%{CYRUS_IMAPD_LMTPUNIX_DELIVERY}|%{CYRUS_IMAPD_LONGLOCK}|%{CYRUS_IMAPD_LMTPUNIX_SIEVE}|%{CYRUS_IMAPD_MAILBOX_EVENTS}|%{CYRUS_IMAPD_SESSION_STATS}|%{CYRUS_IMAPD_SKIPLIST}|%{CYRUS_IMAPD_USAGE}) CYRUS_IMAPD_MASTER_MESSAGES (?:%{CYRUS_IMAPD_MASTER_ERRORS}|%{CYRUS_IMAPD_MASTER_STATUS}) CYRUS_IMAPD_MUPDATE_MESSAGES (?:%{CYRUS_IMAPD_CERT_EXPIRED}|%{CYRUS_IMAPD_MUPDATE_SUCCESS}|%{CYRUS_IMAPD_SCARF}|%{CYRUS_IMAPD_MUPDATE_SYNC}|%{CYRUS_IMAPD_MUPDATE_SYNCOK}|%{CYRUS_IMAPD_MUPDATE_UNREADY}) CYRUS_IMAPD_NOTIFYD_MESSAGES (?:%{CYRUS_IMAPD_EVENT}) CYRUS_IMAPD_POP3_MESSAGES (?:%{BADLOGIN_REASONS}|%{CYRUS_IMAPD_AUDITLOG}|%{CYRUS_IMAPD_CERT_EXPIRED}|%{CYRUS_IMAPD_DHPARAMS}|%{CYRUS_IMAPD_EXPUNGE}|%{CYRUS_IMAPD_LOGIN}|%{CYRUS_IMAPD_LONGLOCK}|%{CYRUS_IMAPD_POP_COUNTS}|%{CYRUS_IMAPD_POP_FAILED}|%{CYRUS_IMAPD_SSLTLS}) CYRUS_IMAPD_POP3S_MESSAGES (?:%{CYRUS_IMAPD_POP3_MESSAGES}) CYRUS_IMAPD_SIEVE_MESSAGES (?:%{CYRUS_IMAPD_LOGIN}|%{CYRUS_IMAPD_SHUTDOWN}|%{CYRUS_IMAPD_SIEVE_DISCONNECT}|%{CYRUS_IMAPD_SSLTLS}) CYRUS_IMAPD_SQUATTER_MESSAGES (?:%{CYRUS_IMAPD_LONGLOCK}|%{CYRUS_IMAPD_MAILBOX_EVENT}|%{CYRUS_SQUATTER}) CYRUS_IMAPD_TLS_PRUNE_MESSAGES (?:%{CYRUS_TLSPRUNE}) diff --git a/docker/logstash/patterns/opendmarc b/docker/logstash/patterns/opendmarc index cd10ab9d..894620d5 100644 --- a/docker/logstash/patterns/opendmarc +++ b/docker/logstash/patterns/opendmarc @@ -1,4 +1,6 @@ +OPENDMARC_IGNORE ignoring connection from \[%{IP}\] +OPENDMARC_IMPLICIT implicit authentication service: %{IP_OR_HOST} OPENDMARC_RESULT (?:%{QUEUEID:local_queueid}: )%{IP_OR_HOST:domain} %{DATA:dmarc_result} OPENDMARC_SPF_RESULT (?:%{QUEUEID:local_queueid}: )SPF\(mailfrom\): %{EMAIL_ADDRESS:from} %{DATA:dmarc_result} -OPENDMARC_MESSAGES (?:(%{OPENDMARC_RESULT}|%{OPENDMARC_SPF_RESULT})) +OPENDMARC_MESSAGES (?:(%{OPENDMARC_IGNORE}|%{OPENDMARC_IMPLICIT}|%{OPENDMARC_RESULT}|%{OPENDMARC_SPF_RESULT})) diff --git a/docker/logstash/patterns/postfix b/docker/logstash/patterns/postfix index e5d21af4..d185577e 100644 --- a/docker/logstash/patterns/postfix +++ b/docker/logstash/patterns/postfix @@ -1,78 +1,78 @@ # Postfix stuff based on https://gist.github.com/jbrownsc/4694374: PERMERROR 5[0-9]{2} MESSAGELEVEL reject|warning|error|fatal|panic POSTFIX_ACTION discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn # Postfix # Queue IDs are hexadecimal, or magic NOQUEUE QUEUEID (?:[A-F0-9]+|NOQUEUE) RELAY (?:%{RELAY_IP}|%{RELAY_SOCKET}) RELAY_IP (?:%{HOSTNAME:relay_host})?\[(?:%{IP:relay_ip})?\](?::%{POSINT})? RELAY_SOCKET (?:%{HOSTNAME:relay_host})?\[(?:%{PATH:relay_socket})?\] POSREAL [0-9]+(.[0-9]+)? DELAYS (%{POSREAL}[/]*)+ DSN %{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT} STATUS sent|deferred|bounced|expired # Permanent errors PERMERROR 5[0-9]{2} # Temporary problems TEMPERROR 4[0-9]{2} POSTFIX_ACTION discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn INVALID_CERT_REASONS (?:certificate has expired|self-signed certificate|untrusted issuer %{DATA:cert_issuer}) # These occur on outgoing connections, as in postfix/smtp, but also postfix/lmtp POSTFIX_4XX (%{POSTFIX_4XX1}|%{POSTFIX_4XX2}) POSTFIX_4XX1 %{QUEUEID:local_queueid}: host %{RELAY} said: %{GREEDYDATA:reason} POSTFIX_4XX2 %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=%{RELAY},(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \(host (?:%{HOSTNAME})?\[(?:%{IP}|%{PATH})?\](?::%{POSINT})? said: %{TEMPERROR:responsecode} %{DATA:smtp_response} \(in reply to %{DATA:command} command\)\) POSTFIX_4XXd NOQUEUE: reject: %{GREEDYDATA:command} from %{CLIENT}: %{POSINT:error_code} %{DSN} (?:<%{DATA}>: )?%{DATA:reason}; from=<%{DATA:from}> to=%{DATA:to}> proto=%{WORD} helo=<%{DATA:smtp_helo}> POSTFIX_5XX %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=(?:%{CLIENT_DEST}|%{RELAY_SOCKET}),(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \(host %{RELAY} said: %{PERMERROR:responsecode} %{DATA:smtp_response} \(in reply to %{DATA:command} command\)\) POSTFIX_CERTINVALID certificate verification failed for %{RELAY}: %{INVALID_CERT_REASONS} POSTFIX_CONNECT_FROM connect from %{CLIENT} POSTFIX_CONNECT_TO connect to %{RELAY}: %{GREEDYDATA:reason} POSTFIX_HOST_STARTTLS Host offered STARTTLS: \[%{IP_OR_HOST}\] POSTFIX_LOST_CONNECTION %{QUEUEID:local_queueid}: lost connection with %{RELAY} while %{GREEDYDATA:reason} # Postfix informational message about CISCO PIX SMTP relay "fixup" protocol POSTFIX_PIX %{QUEUEID:local_queueid}: enabling PIX workarounds: disable_esmtp delay_dotcrlf for %{RELAY} POSTFIX_REFUSAL %{QUEUEID:local_queueid}: host %{RELAY} refused to talk to me: %{GREEDYDATA:reason} -POSTFIX_RELAY %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=(?:%{RELAY}|local|none),(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \((?:.* SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>\))?(?:.* (forwarded|queued) as %{QUEUEID:remote_queueid}\))?(?:%{DATA:reason}\))? +POSTFIX_RELAY %{QUEUEID:local_queueid}: to=<%{EMAIL_ADDRESS:to}>,(?:\sorig_to=<%{EMAIL_ADDRESS:orig_to}>,)? relay=(?:%{RELAY}|local|none),(?:\sconn_use=%{POSINT:conn_use},)? delay=%{POSREAL:delay}, delays=%{DELAYS:delays}, dsn=%{DSN:dsn}, status=%{STATUS:result} \((?:.* SESSIONID=<%{CYRUS_IMAP_SESSIONID:session_id}>\))?(?:.* (forwarded|queued) as %{QUEUEID:remote_queueid}\))?(?:connect to %{CLIENT_DEST}: %{DATA:reason})?(?:%{DATA:reason}\))? POSTFIX_TIMEOUT %{QUEUEID:local_queueid}: conversation with %{RELAY} timed out while %{GREEDYDATA:reason} POSTFIX_TLS_HANDSHAKE %{QUEUEID:local_queueid}: Cannot start TLS: handshake failure POSTFIX_TLS_PROBLEM %{MESSAGELEVEL}: TLS library problem: %{GREEDYDATA} POSTFIX_SSL_ERROR SSL_(?:accept|connect) error (?:from %{CLIENT_SRC}|to %{CLIENT_DEST}): %{DATA} # Use DATA:to, because people mistype address delimiters, and programmers sometimes... well. POSTFIX_ACTIONS %{QUEUEID:local_queueid}: %{POSTFIX_ACTION}: %{DATA:command} from %{CLIENT}: %{GREEDYDATA:reason}; (from=<(?:%{EMAIL_ADDRESS:from})?> (to=<%{DATA:to}> )?)?proto=%{DATA:proto} helo=<%{DATA:smtp_helo}> POSTFIX_AUTH_CLIENT (?:(%{QUEUEID:local_queueid}: client=%{CLIENT}(, sasl_method=%{DATA}, sasl_username=%{EMAIL_ADDRESS:client_auth})?)|warning: %{CLIENT}: SASL PLAIN authentication failed: authentication failure) POSTFIX_CONNECTS (lost connection after %{DATA}|(?:dis)?connect) from %{CLIENT} POSTFIX_TIMEOUTS timeout after %{DATA:command} from %{CLIENT} POSTFIX_CLEANUP %{QUEUEID:local_queueid}: (?:(?:resent-)?message-id=(<%{DATA:message_id}>)?|%{POSTFIX_ACTION}: %{DATA:header_replace} from %{CLIENT}; from=<(%{EMAIL_ADDRESS:from})?> to=<%{EMAIL_ADDRESS:to}> proto=%{DATA} helo=<%{DATA:smtp_helo}>: %{GREEDYDATA:header_replacement}) POSTFIX_DSN_NDN %{QUEUEID:local_queueid}: sender (delivery status|non\-delivery) notification: %{QUEUEID:remote_queueid} POSTFIX_PICKUP %{QUEUEID:local_queueid}: uid=\d+ from=<%{DATA:from}> POSTFIX_QMGR %{QUEUEID:local_queueid}: (?:removed|from=<(?:%{EMAIL_ADDRESS:from})?>(?:, size=%{POSINT:size}, nrcpt=%{POSINT:nrcpt} \(%{GREEDYDATA:queuestatus}\))?) POSTFIX_TOO_MANY_ERRORS too many errors after %{DATA:command} from %{CLIENT} POSTFIX_IMPROPER_COMMAND_PIPELINING improper command pipelining after %{DATA:command} from %{CLIENT}%{DATA} POSTFIX_STATS statistics: %{GREEDYDATA} POSTFIX_WARNING warning: %{GREEDYDATA} POSTFIX_ANVIL_MESSAGES (?:%{POSTFIX_STATS}) POSTFIX_BOUNCE_MESSAGES (?:%{POSTFIX_DSN_NDN}) POSTFIX_CLEANUP_MESSAGES (?:%{POSTFIX_CLEANUP}) POSTFIX_ERROR_MESSAGES (?:%{POSTFIX_RELAY}) POSTFIX_LMTP_MESSAGES (?:%{POSTFIX_RELAY}) POSTFIX_LOCAL_MESSAGES (?:%{POSTFIX_RELAY}|%{POSTFIX_WARNING}) POSTFIX_MASTER_MESSAGES (?:%{POSTFIX_WARNING}|terminating on signal \d+|daemon started -- version %{DATA}, configuration %{PATH}) POSTFIX_PICKUP_MESSAGES (?:%{POSTFIX_PICKUP}) POSTFIX_QMGR_MESSAGES (?:%{POSTFIX_QMGR}|%{POSTFIX_RELAY}|%{POSTFIX_WARNING}) POSTFIX_SCACHE_MESSAGES (?:%{POSTFIX_STATS}) POSTFIX_SCRIPT_MESSAGES (?:starting|stopping|waiting for) the Postfix mail system(%{DATA})? POSTFIX_SMTP_MESSAGES (?:%{POSTFIX_4XX}|%{POSTFIX_5XX}|%{POSTFIX_CERTINVALID}|%{POSTFIX_CONNECT_FROM}|%{POSTFIX_CONNECT_TO}|%{POSTFIX_HOST_STARTTLS}|%{POSTFIX_LOST_CONNECTION}|%{POSTFIX_PIX}|%{POSTFIX_REFUSAL}|%{POSTFIX_RELAY}|%{POSTFIX_TIMEOUT}|%{POSTFIX_TLS_HANDSHAKE}|%{POSTFIX_TLS_PROBLEM}|%{POSTFIX_SSL_ERROR}) POSTFIX_SMTPD_MESSAGES (?:%{POSTFIX_4XXd}|%{POSTFIX_ACTIONS}|%{POSTFIX_AUTH_CLIENT}|%{POSTFIX_CONNECTS}|%{POSTFIX_SSL_ERROR}|%{POSTFIX_TIMEOUTS}|%{POSTFIX_TOO_MANY_ERRORS}|%{POSTFIX_WARNING}|%{POSTFIX_IMPROPER_COMMAND_PIPELINING}) POSTFIX_SPAWN_MESSAGES (?:fatal|warning): %{GREEDYDATA:reason} POSTFIX_TRIVIAL_REWRITE_MESSAGES (?:%{POSTFIX_WARNING}) diff --git a/docker/logstash/patterns/sqlgrey b/docker/logstash/patterns/sqlgrey new file mode 100644 index 00000000..c361d615 --- /dev/null +++ b/docker/logstash/patterns/sqlgrey @@ -0,0 +1,20 @@ +# Example sequence: +# grey: new: w.x.y(w.x.y.z), local@sender-domain.com -> local@recipient-domain.com +# grey: early reconnect: x.y.z(w.x.y.z), local@sender-domain.com- -> local@recipient-domain.com +# grey: reconnect ok: w.x.y(w.x.y.z), local@sender-domain.com -> local@recipient-domain.com (00:05:59) +# grey: from awl: w.x.y, local@sender-domain.com added +# grey: from awl match: updating w.x.y(w.x.y.z), local@sender-domain.com(local@sender-domain.com) +# grey: domain awl match: updating w.x.y(w.x.y.z), sender-domain.com +# whitelist: local@sender-domain.com, w.x.y.z(sending.mta.domain.com) -> local@recipient-domain.com +# perf: spent 0s cleaning: from_awl (0) domain_awl (6) connect (13) +# spam: w.x.y: local@sender-domain.com -> local@recipient-domain.com at YYYY-mm-dd HH:MM:SS +SQLGREY_AWL_ADDED grey: (domain|from) awl: %{IPV4_3OCT}, (?:(%{HOSTNAME}|%{EMAIL_ADDRESS:from}|%{DATA:from})) added +SQLGREY_AWL_MATCH grey: (?:(domain|from)) awl match: updating %{IPV4_3OCT}\(%{IPV4:src_ip}\), (?:(%{EMAIL_ADDRESS_DOMAIN:domain}|%{DATA:from}\(%{DATA}\))) +SQLGREY_NEW grey: new: %{IPV4_3OCT}\(%{IPV4:src_ip}\), %{DATA:from} -> %{EMAIL_ADDRESS:to} +SQLGREY_PERF perf: %{GREEDYDATA} +SQLGREY_RECONNECT_EARLY grey: early reconnect: %{IPV4_3OCT}\(%{IPV4:src_ip}\), (?:%{EMAIL_ADDRESS:from}|%{DATA}) -> %{EMAIL_ADDRESS:to} +SQLGREY_RECONNECT_OK grey: reconnect ok: %{IPV4_3OCT}\(%{IPV4:src_ip}\), %{DATA:from} -> %{EMAIL_ADDRESS:to} \(%{HOUR}:%{MINUTE}:%{SECOND}\) +SQLGREY_SPAM spam: %{IPV4_3OCT}: %{DATA} -> %{EMAIL_ADDRESS:to} at %{TIMESTAMP_ISO8601} +SQLGREY_WHITELIST whitelist: %{DATA:from}, %{IPV4:src_ip}\(%{HOSTNAME:client_host}\) -> %{EMAIL_ADDRESS:to} + +SQLGREY_MESSAGES (?:(%{SQLGREY_AWL_ADDED}|%{SQLGREY_AWL_MATCH}|%{SQLGREY_NEW}|%{SQLGREY_PERF}|%{SQLGREY_RECONNECT_EARLY}|%{SQLGREY_RECONNECT_OK}|%{SQLGREY_SPAM}|%{SQLGREY_WHITELIST})) diff --git a/docker/logstash/pipeline/10_input_filesystem_maillog.conf b/docker/logstash/pipeline/10_input_filesystem_maillog.conf index 3d9387a0..8f555dae 100644 --- a/docker/logstash/pipeline/10_input_filesystem_maillog.conf +++ b/docker/logstash/pipeline/10_input_filesystem_maillog.conf @@ -1,9 +1,10 @@ +# Gather plenty of data from /data/logs/ input { file { exclude => [ "*.gz", "*.orig" ] path => "/data/logs/maillog-*" sincedb_path => "/usr/share/logstash/.sincedb-maillog" start_position => beginning type => "maillog" } } diff --git a/docker/logstash/pipeline/10_input_redis_suricata.conf b/docker/logstash/pipeline/10_input_redis_suricata.conf new file mode 100644 index 00000000..737ce212 --- /dev/null +++ b/docker/logstash/pipeline/10_input_redis_suricata.conf @@ -0,0 +1,9 @@ +input { + redis { + data_type => "list" + host => "127.0.0.1" + key => "suricata" + port => 6379 + type => "suricata" + } +} diff --git a/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf b/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf index 84957fbb..540340ba 100644 --- a/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf +++ b/docker/logstash/pipeline/12_input_elasticsearch_grokparsefailure.conf @@ -1,7 +1,8 @@ +# Retrieves entries in elasticsearch marked as grok parse failures input { elasticsearch { index => "logstash-*" query => '{"query": {"match": {"tags":"_grokparsefailure"}}}' tags => [ "elasticsearch" ] } } diff --git a/docker/logstash/pipeline/24_filter_system_tags.conf b/docker/logstash/pipeline/24_filter_system_tags.conf new file mode 100644 index 00000000..7ab151fb --- /dev/null +++ b/docker/logstash/pipeline/24_filter_system_tags.conf @@ -0,0 +1,17 @@ +filter { + if [host] { + if [host] =~ /^ext-imapf[0-9]+$/ { + mutate { + add_tag => "sys-in" + } + } else if [host] =~ /^ext-mx-in[0-9]+$/ { + mutate { + add_tag => "sys-in" + } + } else if [host] =~ /^ext-mx-out[0-9]+$/ { + mutate { + add_tag => "sys-out" + } + } + } +} diff --git a/docker/logstash/pipeline/30_filter_maillog.conf b/docker/logstash/pipeline/30_filter_maillog.conf index 4bdab1cb..5c4a9696 100644 --- a/docker/logstash/pipeline/30_filter_maillog.conf +++ b/docker/logstash/pipeline/30_filter_maillog.conf @@ -1,278 +1,314 @@ filter { if [type] == "maillog" { if [syslog_program] == "ctl_cyrusdb" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "ctl_mboxlist" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "cyr_expire" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "cyr_info" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "imap" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } # Specialty MyKolab -> Kolab Now item } else if [syslog_program] == "imapkn" { mutate { replace => [ "syslog_program", "cyrus-imapd/imap" ] } } else if [syslog_program] == "imaps" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } # Specialty MyKolab -> Kolab Now item } else if [syslog_program] == "imapskn" { mutate { replace => [ "syslog_program", "cyrus-imapd/imaps" ] } } else if [syslog_program] == "lmtpunix" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "pop3" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "pop3s" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "squatter" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } else if [syslog_program] == "tls_prune" { mutate { replace => [ "syslog_program", "cyrus-imapd/%{syslog_program}" ] } } + #if [syslog_program] == "cyrus-imapd/imaps" { + # if [message] =~ /^(bad)?login:/ { + # grok { + # match => [ "message", "%{CYRUS_IMAPD_LOGIN}" ] + # patterns_dir => "/usr/share/logstash/patterns/" + # } + # } else { + # drop { } + # } + #} else { + # drop { } + #} + if [syslog_program] =~ /^amavis$/ { if [message] =~ / (Passed|Blocked) / { grok { match => [ "message", "%{AMAVIS_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { drop { } } } else if [syslog_program] =~ /^cyrus-imapd\// { if [message] =~ /^getaddrinfo(.*) failed:/ { drop { } + } else if [message] =~ /Header CRC mismatch/ { + drop { } } else if [syslog_program] == "cyrus-imapd/ctl_cyrusdb" { grok { match => [ "message", "%{CYRUS_IMAPD_CTL_CYRUSDB_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/ctl_mboxlist" { grok { match => [ "message", "%{CYRUS_IMAPD_CTL_MBOXLIST_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/cyr_expire" { grok { match => [ "message", "%{CYRUS_IMAPD_CYR_EXPIRE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/cyr_info" { grok { match => [ "message", "%{CYRUS_IMAPD_CYR_INFO_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/imap" { grok { match => [ "message", "%{CYRUS_IMAPD_IMAP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/imaps" { grok { match => [ "message", "%{CYRUS_IMAPD_IMAPS_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/lmtpd" { grok { match => [ "message", "%{CYRUS_IMAPD_LMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/lmtpunix" { grok { match => [ "message", "%{CYRUS_IMAPD_LMTPUNIX_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/master" { grok { match => [ "message", "%{CYRUS_IMAPD_MASTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/mupdate" { grok { match => [ "message", "%{CYRUS_IMAPD_MUPDATE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/notifyd" { grok { match => [ "message", "%{CYRUS_IMAPD_NOTIFYD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/pop3" { grok { match => [ "message", "%{CYRUS_IMAPD_POP3_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/pop3s" { grok { match => [ "message", "%{CYRUS_IMAPD_POP3S_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/timsieved" { grok { match => [ "message", "%{CYRUS_IMAPD_SIEVE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "cyrus-imapd/squatter" { grok { match => [ "message", "%{CYRUS_IMAPD_SQUATTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } + } else if [syslog_program] == "cyrus-imapd/sync_client" { + drop { } + } else if [syslog_program] == "cyrus-imapd/syncserver" { + drop { } } else if [syslog_program] == "cyrus-imapd/tls_prune" { grok { match => [ "message", "%{CYRUS_IMAPD_TLS_PRUNE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { mutate { add_tag => [ "_unparsed" ] } } } else if [syslog_program] == "opendmarc" { grok { match => [ "message", "%{OPENDMARC_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] =~ /^postfix\// { if [message] =~ /fatal:.*table lookup problem/ { drop { } } else if [syslog_program] == "postfix/anvil" { grok { match => [ "message", "%{POSTFIX_ANVIL_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/bounce" { grok { match => [ "message", "%{POSTFIX_BOUNCE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/cleanup" { grok { match => [ "message", "%{POSTFIX_CLEANUP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/error" { grok { match => [ "message", "%{POSTFIX_ERROR_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/lmtp" { grok { match => [ "message", "%{POSTFIX_LMTP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/local" { grok { match => [ "message", "%{POSTFIX_LOCAL_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/master" { grok { match => [ "message", "%{POSTFIX_MASTER_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/pickup" { grok { match => [ "message", "%{POSTFIX_PICKUP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/postfix-script" { grok { match => [ "message", "%{POSTFIX_SCRIPT_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/qmgr" { grok { match => [ "message", "%{POSTFIX_QMGR_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/scache" { grok { match => [ "message", "%{POSTFIX_SCACHE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] =~ /^postfix(\/(polite|turtle))?\/smtp$/ { grok { match => [ "message", "%{POSTFIX_SMTP_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } if [relay_host] { mutate { lowercase => [ "relay_host" ] } } if [relay_ip] { mutate { copy => { "relay_ip" => "dest_ip" } } } } else if [syslog_program] == "postfix/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/smtps/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/submission/smtpd" { grok { match => [ "message", "%{POSTFIX_SMTPD_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/spawn" { grok { match => [ "message", "%{POSTFIX_SPAWN_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else if [syslog_program] == "postfix/trivial-rewrite" { grok { match => [ "message", "%{POSTFIX_TRIVIAL_REWRITE_MESSAGES}" ] patterns_dir => "/usr/share/logstash/patterns/" } } else { mutate { add_tag => [ "_unparsed" ] } } - } else { + } + + else if [syslog_program] == "sqlgrey" { + grok { + match => [ "message", "%{SQLGREY_MESSAGES}" ] + patterns_dir => "/usr/share/logstash/patterns/" + } + + if [from] { + if [from] == "-undef-@-undef-" { + mutate { + remove_field => [ "from" ] + } + } + } + } + + else { mutate { add_tag => [ "_unparsed" ] } } } } diff --git a/docker/logstash/pipeline/31_filter_suricata.conf b/docker/logstash/pipeline/31_filter_suricata.conf new file mode 100644 index 00000000..99810850 --- /dev/null +++ b/docker/logstash/pipeline/31_filter_suricata.conf @@ -0,0 +1,14 @@ +filter { + if [type] == "suricata" { + date { + match => [ "timestamp", "ISO8601" ] + target => "@timestamp" + } + } + + if [timestamp] { + mutate { + remove_field => [ "timestamp" ] + } + } +} diff --git a/docker/logstash/pipeline/40_network.conf b/docker/logstash/pipeline/40_network.conf index 622f00e7..7a3d0410 100644 --- a/docker/logstash/pipeline/40_network.conf +++ b/docker/logstash/pipeline/40_network.conf @@ -1,158 +1,208 @@ filter { if [dest_ip] { cidr { address => [ "%{dest_ip}" ] network => [ "0::/16", # unknown "fc00::/7", # local private network range "fe80::/10", # link-local "ff00::/16", # reserved "ff01::/16", # interface-local "ff02::/16", # link-local "ff03::/16", # IPv4 local scope "ff04::/16", # admin-local "ff05::/16", # site-local "ff06::/16", # reserved "ff07::/16", # reserved "ff08::/16", # organization-local "ff09::/16", # reserved "ff0a::/16", # reserved "ff0b::/16", # reserved "ff0c::/16", # reserved "ff0d::/16", # reserved "ff0e::/16", # global scope "ff0f::/16" # reserved ] add_tag => [ "domestic_dest" ] } cidr { address => [ "%{dest_ip}" ] network => [ "0.0.0.0/32", # unknown "224.0.0.0/24", # local subnetwork "224.0.1.0/24", # internetwork control "224.0.2.0/24", # ad-hoc block 1 "224.3.0.0/16", # ad-hoc block 2 "224.4.0.0/16", # ad-hoc block 2 "232.0.0.0/8", # source-specific multicast "233.252.0.0/6", # ad-hoc block 3 "234.0.0.0/8", # unicast-prefix based "239.0.0.0/8", # administratively scoped "255.255.255.255/32" # broadcast ] add_tag => [ "domestic_dest" ] } cidr { address => [ "%{dest_ip}" ] network => [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" ] add_tag => [ "domestic_dest" ] } cidr { add_tag => [ "localhost" ] address => [ "%{dest_ip}" ] network => [ "127.0.0.0/8" ] } + + if "domestic_dest" not in [tags] { + if "localhost" not in [tags] { + mutate { + add_tag => "foreign_dest" + } + } + } } if [src_ip] { cidr { address => [ "%{src_ip}" ] network => [ "0::/16", # unknown "fc00::/7", # local private network range "fe80::/10", # link-local "ff00::/16", # reserved "ff01::/16", # interface-local "ff02::/16", # link-local "ff03::/16", # IPv4 local scope "ff04::/16", # admin-local "ff05::/16", # site-local "ff06::/16", # reserved "ff07::/16", # reserved "ff08::/16", # organization-local "ff09::/16", # reserved "ff0a::/16", # reserved "ff0b::/16", # reserved "ff0c::/16", # reserved "ff0d::/16", # reserved "ff0e::/16", # global scope "ff0f::/16" # reserved ] add_tag => [ "domestic_src" ] } cidr { address => [ "%{src_ip}" ] network => [ "0.0.0.0/32", # unknown "224.0.0.0/24", # local subnetwork "224.0.1.0/24", # internetwork control "224.0.2.0/24", # ad-hoc block 1 "224.3.0.0/16", # ad-hoc block 2 "224.4.0.0/16", # ad-hoc block 2 "232.0.0.0/8", # source-specific multicast "233.252.0.0/6", # ad-hoc block 3 "234.0.0.0/8", # unicast-prefix based "239.0.0.0/8", # administratively scoped "255.255.255.255/32" # broadcast ] add_tag => [ "domestic_src" ] } cidr { address => [ "%{src_ip}" ] network => [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" ] add_tag => [ "domestic_src" ] } cidr { add_tag => [ "localhost" ] address => [ "%{src_ip}" ] network => [ "127.0.0.0/8" ] } + + if "domestic_src" not in [tags] { + if "localhost" not in [tags] { + mutate { + add_tag => "foreign_src" + } + } + } } if [dest_ip] or [src_ip] { if "localhost" in [tags] { mutate { - remove_tag => [ "domestic_dest", "domestic_src" ] + remove_tag => [ + "domestic_dest", + "domestic_src", + "foreign_dest", + "foreign_src", + "sys-in", + "sys-out" + ] } } else { if "domestic_dest" in [tags] { if "domestic_src" in [tags] { mutate { add_tag => [ "domestic" ] remove_tag => [ "domestic_dest", "domestic_src" ] } - } else { + } else if "foreign_src" in [tags] { mutate { add_tag => [ "inbound" ] - remove_tag => [ "domestic_dest" ] + remove_tag => [ "domestic_dest", "foreign_src" ] + } + } else { + mutate { + add_tag => [ "domestic" ] + remove_tag => [ "domestic_dest", "sys-in" ] } } - } else { - if "domestic_src" in [tags] { + } else if "domestic_src" in [tags] { + if "foreign_dest" in [tags] { mutate { add_tag => [ "outbound" ] - remove_tag => [ "domestic_src" ] + remove_tag => [ "domestic_src", "foreign_dest" ] } } else { + mutate { + add_tag => [ "domestic" ] + remove_tag => [ "domestic_src", "sys-in", "sys-out" ] + } + } + } else if "foreign_dest" in [tags] { + if "foreign_src" in [tags] { mutate { add_tag => [ "foreign" ] + remove_tag => [ "foreign_src", "foreign_dest" ] + } + } else if "sys-out" in [tags] { + mutate { + add_tag => [ "outbound" ] + remove_tag => [ "foreign_dest", "sys-out" ] } } + } else if "foreign_src" in [tags] { + mutate { + add_tag => [ "inbound" ] + remove_tag => [ "foreign_src", "sys-in" ] + } } } } + + mutate { + remove_tag => [ "sys-in", "sys-out" ] + } } diff --git a/docker/logstash/pipeline/91_output_elasticsearch_suricata.conf b/docker/logstash/pipeline/91_output_elasticsearch_suricata.conf new file mode 100644 index 00000000..61c8e0a8 --- /dev/null +++ b/docker/logstash/pipeline/91_output_elasticsearch_suricata.conf @@ -0,0 +1,31 @@ +output { + if [type] == "suricata" { + if [event_type] == "flow" { + elasticsearch { + hosts => [ "127.0.0.1:9200" ] + index => "suricata-flow-1.0.0-%{+YYYY.MM.dd}" + template => "/usr/share/logstash/templates/suricata-flow_template.json" + template_name => "suricata-flow-1.0.0" + template_overwrite => true + } + } else if [event_type] == "netflow" { + elasticsearch { + hosts => [ "127.0.0.1:9200" ] + index => "suricata-netflow-1.0.0-%{+YYYY.MM.dd}" + } + } else if [event_type] == "stats" { + elasticsearch { + hosts => [ "127.0.0.1:9200" ] + index => "suricata-stats-1.0.0-%{+YYYY.MM.dd}" + } + } else { + elasticsearch { + hosts => [ "127.0.0.1:9200" ] + index => "suricata-1.0.0-%{+YYYY.MM.dd}" + template => "/usr/share/logstash/templates/suricata_template.json" + template_name => "suricata-1.0.0" + template_overwrite => "true" + } + } + } +} diff --git a/docker/logstash/pipeline/91_output_stdout_grokparsefailure.conf b/docker/logstash/pipeline/97_output_stdout_grokparsefailure.conf similarity index 100% rename from docker/logstash/pipeline/91_output_stdout_grokparsefailure.conf rename to docker/logstash/pipeline/97_output_stdout_grokparsefailure.conf diff --git a/docker/logstash/pipeline/92_output_file_grokparsefailure.conf b/docker/logstash/pipeline/98_output_file_grokparsefailure.conf similarity index 100% rename from docker/logstash/pipeline/92_output_file_grokparsefailure.conf rename to docker/logstash/pipeline/98_output_file_grokparsefailure.conf diff --git a/docker/logstash/templates/suricata-flow_template.json b/docker/logstash/templates/suricata-flow_template.json new file mode 100644 index 00000000..f6002aaf --- /dev/null +++ b/docker/logstash/templates/suricata-flow_template.json @@ -0,0 +1,330 @@ +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "app_proto": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "dest_ip": { + "type": "ip" + }, + "dest_port": { + "type": "long" + }, + "event_type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "flow": { + "properties": { + "age": { + "type": "long" + }, + "alerted": { + "type": "boolean" + }, + "bytes_toclient": { + "type": "long" + }, + "bytes_toserver": { + "type": "long" + }, + "end": { + "type": "date" + }, + "pkts_toclient": { + "type": "long" + }, + "pkts_toserver": { + "type": "long" + }, + "reason": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "start": { + "type": "date" + }, + "state": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "flow_id": { + "type": "long" + }, + "geoip_dest": { + "dynamic": true, + "properties": { + "city_name": { + "type": "text" + }, + "country_code2": { + "type": "text" + }, + "country_code3": { + "type": "text" + }, + "continent_code": { + "type": "text" + }, + "dma_code": { + "type": "integer" + }, + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + }, + "postal_code": { + "type": "text" + }, + "region_code": { + "type": "text" + }, + "region_name": { + "type": "text" + }, + "timezone": { + "type": "text" + } + } + }, + "geoip_src": { + "dynamic": true, + "properties": { + "city_name": { + "type": "text" + }, + "country_code2": { + "type": "text" + }, + "country_code3": { + "type": "text" + }, + "continent_code": { + "type": "text" + }, + "dma_code": { + "type": "integer" + }, + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + }, + "postal_code": { + "type": "text" + }, + "region_code": { + "type": "text" + }, + "region_name": { + "type": "text" + }, + "timezone": { + "type": "text" + } + } + }, + "host": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "icmp_code": { + "type": "long" + }, + "icmp_type": { + "type": "long" + }, + "location_dest": { + "type": "geo_point" + }, + "location_src": { + "type": "geo_point" + }, + "metadata": { + "properties": { + "flowbits": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "flowints": { + "properties": { + "tcp": { + "properties": { + "retransmission": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "proto": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "response_icmp_code": { + "type": "long" + }, + "response_icmp_type": { + "type": "long" + }, + "src_ip": { + "type": "ip" + }, + "src_port": { + "type": "long" + }, + "tags": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "tcp": { + "properties": { + "ack": { + "type": "boolean" + }, + "fin": { + "type": "boolean" + }, + "psh": { + "type": "boolean" + }, + "rst": { + "type": "boolean" + }, + "state": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "syn": { + "type": "boolean" + }, + "tcp_flags": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "tcp_flags_tc": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "tcp_flags_ts": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "order": 1, + "settings": { + "index": { + "refresh_interval": "30s" + } + }, + "template": "suricata-flow-1.0.0-*", + "version": 70001 +} diff --git a/docker/logstash/templates/suricata_template.json b/docker/logstash/templates/suricata_template.json new file mode 100644 index 00000000..aab66c2f --- /dev/null +++ b/docker/logstash/templates/suricata_template.json @@ -0,0 +1,697 @@ +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "alert": { + "properties": { + "action": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "category": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "gid": { + "type": "long" + }, + "rev": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "signature": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "signature_id": { + "type": "long" + } + } + }, + "app_proto": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "dest_ip": { + "type": "ip" + }, + "dest_port": { + "type": "long" + }, + "dns": { + "properties": { + "answers": { + "properties": { + "rdata": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "rrname": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "rrtype": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "ttl": { + "type": "long" + } + } + }, + "flags": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "grouped": { + "properties": { + "A": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "AAAA": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "id": { + "type": "long" + }, + "qr": { + "type": "boolean" + }, + "ra": { + "type": "boolean" + }, + "rcode": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "rd": { + "type": "boolean" + }, + "rrname": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "rrtype": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "tx_id": { + "type": "long" + }, + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "version": { + "type": "long" + } + } + }, + "drop": { + "properties": { + "ack": { + "type": "boolean" + }, + "fin": { + "type": "boolean" + }, + "ipid": { + "type": "long" + }, + "len": { + "type": "long" + }, + "psh": { + "type": "boolean" + }, + "rst": { + "type": "boolean" + }, + "syn": { + "type": "boolean" + }, + "tcpack": { + "type": "long" + }, + "tcpres": { + "type": "long" + }, + "tcpseq": { + "type": "long" + }, + "tcpurgp": { + "type": "long" + }, + "tcpwin": { + "type": "long" + }, + "tos": { + "type": "long" + }, + "ttl": { + "type": "long" + }, + "urg": { + "type": "boolean" + } + } + }, + "event_type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "fileinfo": { + "properties": { + "filename": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "gaps": { + "type": "boolean" + }, + "size": { + "type": "long" + }, + "state": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "stored": { + "type": "boolean" + }, + "tx_id": { + "type": "long" + } + } + }, + "flow": { + "properties": { + "bytes_toclient": { + "type": "long" + }, + "bytes_toserver": { + "type": "long" + }, + "pkts_toclient": { + "type": "long" + }, + "pkts_toserver": { + "type": "long" + }, + "start": { + "type": "date" + } + } + }, + "flow_id": { + "type": "long" + }, + "geoip_dest": { + "dynamic": true, + "properties": { + "city_name": { + "type": "text" + }, + "country_code2": { + "type": "text" + }, + "country_code3": { + "type": "text" + }, + "continent_code": { + "type": "text" + }, + "dma_code": { + "type": "integer" + }, + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + }, + "postal_code": { + "type": "text" + }, + "region_code": { + "type": "text" + }, + "region_name": { + "type": "text" + }, + "timezone": { + "type": "text" + } + } + }, + "geoip_src": { + "dynamic": true, + "properties": { + "city_name": { + "type": "text" + }, + "country_code2": { + "type": "text" + }, + "country_code3": { + "type": "text" + }, + "continent_code": { + "type": "text" + }, + "dma_code": { + "type": "integer" + }, + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + }, + "postal_code": { + "type": "text" + }, + "region_code": { + "type": "text" + }, + "region_name": { + "type": "text" + }, + "timezone": { + "type": "text" + } + } + }, + "host": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "http": { + "properties": { + "hostname": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "http_content_type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "http_method": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "http_port": { + "type": "long" + }, + "http_refer": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "http_user_agent": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "length": { + "type": "long" + }, + "protocol": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "redirect": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "status": { + "type": "long" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "icmp_code": { + "type": "long" + }, + "icmp_type": { + "type": "long" + }, + "location_dest": { + "type": "geo_point" + }, + "location_src": { + "type": "geo_point" + }, + "packet": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "packet_info": { + "properties": { + "linktype": { + "type": "long" + } + } + }, + "payload": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "proto": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "src_ip": { + "type": "ip" + }, + "src_port": { + "type": "long" + }, + "ssh": { + "properties": { + "client": { + "properties": { + "proto_version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "software_version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "server": { + "properties": { + "proto_version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "software_version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + } + } + }, + "stream": { + "type": "long" + }, + "tags": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "tls": { + "properties": { + "fingerprint": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "issuerdn": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "ja3": { + "type": "object" + }, + "notafter": { + "type": "date" + }, + "notbefore": { + "type": "date" + }, + "serial": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "session_resumed": { + "type": "boolean" + }, + "sni": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "subject": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "tx_id": { + "type": "long" + }, + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "order": 1, + "settings": { + "index": { + "refresh_interval": "30s" + } + }, + "template": "suricata-1.0.0-*", + "version": 70001 +} diff --git a/src/app/Console/Commands/UserLoginsCommand.php b/src/app/Console/Commands/UserLoginsCommand.php index 78300e81..375c3da2 100644 --- a/src/app/Console/Commands/UserLoginsCommand.php +++ b/src/app/Console/Commands/UserLoginsCommand.php @@ -1,85 +1,85 @@ _es = $elasticsearch; } /** * Execute the console command. * * @return mixed */ public function handle() { $this->info("Searching logins for user {$this->argument('userid')}"); $json = sprintf( '{ "from": 0, "size": 1000, "sort": [ { "@timestamp": "desc" } ], "query": { "query_string": { "query": "client_auth:\"%s\" AND tags:(foreign OR inbound)" } } }', $this->argument('userid') ); $params = [ "index" => "logstash-*", "body" => $json ]; $result = $this->_es->search($params); foreach ($result['hits']['hits'] as $item) { $this->info( sprintf( "%s: %s (%s, %s)", $item['_source']['@timestamp'], $item['_source']['src_ip'], $item['_source']['geo_src']['country_name'], - $item['_source']['geo_src']['city_name'] + array_key_exists('city_name', $item['_source']['geo_src']) ? $item['_source']['geo_src']['city_name'] : "" ) ); } $this->info('Done!'); } }