diff --git a/docker/vector/rootfs/config/tests.yaml b/docker/vector/rootfs/config/tests.yaml index 2f60bf21..c24ca558 100644 --- a/docker/vector/rootfs/config/tests.yaml +++ b/docker/vector/rootfs/config/tests.yaml @@ -1,213 +1,239 @@ tests: - name: "Roundcube logfmt output" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-78bd5fccb-lssj6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-78bd5fccb pod_uid: cc681420-f781-4332-b966-73812b372801 - message: name=kolabfiles component=chwala session=a410dfeb user=admin@kolab.local log="Refreshing the access token" + log: name=kolabfiles component=chwala session=a410dfeb user=admin@kolab.local log="Refreshing the access token" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:18:49.006951369Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | assert_eq!(.component, "chwala") assert_eq!(.user, "admin@kolab.local") assert_eq!(.log, "Refreshing the access token") assert!(is_timestamp(.timestamp)) - name: "Roundcube php warning" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-78bd5fccb-lssj6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-78bd5fccb pod_uid: cc681420-f781-4332-b966-73812b372801 - message: 'PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667' + log: 'PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:19:06.317485633Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | - # assert_eq!(.component, "what") - # assert_eq!(.user, "admin@kolab.local") - assert_eq!(.message, "PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667") + assert_eq!(.log, "PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667") assert!(is_timestamp(.timestamp)) - name: "Roundcube generic message" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-74d95fdbf5-9vq5n_7625ef26-43e4-4398-956f-c8b9f10c38e8/roundcube/0.log container_id: containerd://8aaa3192e31ef776307f7ea96d3c8fa71a15494dd22a400ae7d74f3e221fc33b container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-74d95fdbf5-9vq5n pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-74d95fdbf5 pod_uid: 7625ef26-43e4-4398-956f-c8b9f10c38e8 - message: "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)" + log: "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-18T09:18:38.276041789Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | - assert_eq!(.message, "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)") + assert_eq!(.log, "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)") assert!(is_timestamp(.timestamp)) - name: "Roundcube apache" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-74d95fdbf5-9vq5n_7625ef26-43e4-4398-956f-c8b9f10c38e8/roundcube/0.log container_id: containerd://8aaa3192e31ef776307f7ea96d3c8fa71a15494dd22a400ae7d74f3e221fc33b container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-74d95fdbf5-9vq5n pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-74d95fdbf5 pod_uid: 7625ef26-43e4-4398-956f-c8b9f10c38e8 - message: "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"" + log: "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-18T09:18:38.276041789Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | - assert_eq!(.message, "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"") + assert_eq!(.log, "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"") + assert!(is_timestamp(.timestamp)) +- name: "Syncroton" + inputs: + - insert_at: "apps" + type: "log" + log_fields: + file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log + container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a + container_image: localhost:5000/roundcube:latest + container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 + container_name: roundcube + pod_name: roundcube-78bd5fccb-lssj6 + pod_namespace: kolab + pod_node_name: dws07.kolabsys.com + pod_owner: ReplicaSet/roundcube-78bd5fccb + pod_uid: cc681420-f781-4332-b966-73812b372801 + log: name=console component=syncroton cmd=FolderSync device=v140Device type=iphone user=admin@kolab.local log="Syncroton_Server::_handlePost::186 xml response (0):\n\n\n\n 1<\/Status>\n 1<\/SyncKey>\n \n 9<\/Count>\n \n DAV:event:f8a6ef015ca8eb311f00bc145523363da<\/ServerId>\n 0<\/ParentId>\n Calendar<\/DisplayName>\n 8<\/Type>\n <\/Add>\n \n DAV:contact:f027577d9ee8a8fafaf314f1a5db41a81<\/ServerId>\n 0<\/ParentId>\n Addressbook<\/DisplayName>\n 9<\/Type>\n <\/Add>\n \n 38b950ebd62cd9a66929c89615d0fc04<\/ServerId>\n 0<\/ParentId>\n INBOX<\/DisplayName>\n 2<\/Type>\n <\/Add>\n \n 1bb8c55fe84d52c6968db2571f7dc124<\/ServerId>\n 0<\/ParentId>\n Archive<\/DisplayName>\n 12<\/Type>\n <\/Add>\n \n 16833612eebc283ce2fe3c447fb53eff<\/ServerId>\n 0<\/ParentId>\n Drafts<\/DisplayName>\n 3<\/Type>\n <\/Add>\n \n ea8bb554b4246f8a48ccab88b653da54<\/ServerId>\n 0<\/ParentId>\n Sent<\/DisplayName>\n 5<\/Type>\n <\/Add>\n \n 715ed9ea29b8a5377a69c1f758037c65<\/ServerId>\n 0<\/ParentId>\n Spam<\/DisplayName>\n 12<\/Type>\n <\/Add>\n \n 22b8cf366c106d7c8253f4d390f697b8<\/ServerId>\n 0<\/ParentId>\n Trash<\/DisplayName>\n 4<\/Type>\n <\/Add>\n \n DAV:task:f8a6ef015ca8eb311f00bc145523363da<\/ServerId>\n 0<\/ParentId>\n Calendar<\/DisplayName>\n 7<\/Type>\n <\/Add>\n <\/Changes>\n<\/FolderSync>\n" + source_type: kubernetes_logs + stream: stderr + timestamp: "2024-08-20T13:19:46.879013117Z" + outputs: + - extract_from: "parse_roundcube" + conditions: + - type: "vrl" + source: | + assert_eq!(.log, "Syncroton_Server::_handlePost::186 xml response (0):\n\n\n\n 1<\\/Status>\n 1<\\/SyncKey>\n \n 9<\\/Count>\n \n DAV:event:f8a6ef015ca8eb311f00bc145523363da<\\/ServerId>\n 0<\\/ParentId>\n Calendar<\\/DisplayName>\n 8<\\/Type>\n <\\/Add>\n \n DAV:contact:f027577d9ee8a8fafaf314f1a5db41a81<\\/ServerId>\n 0<\\/ParentId>\n Addressbook<\\/DisplayName>\n 9<\\/Type>\n <\\/Add>\n \n 38b950ebd62cd9a66929c89615d0fc04<\\/ServerId>\n 0<\\/ParentId>\n INBOX<\\/DisplayName>\n 2<\\/Type>\n <\\/Add>\n \n 1bb8c55fe84d52c6968db2571f7dc124<\\/ServerId>\n 0<\\/ParentId>\n Archive<\\/DisplayName>\n 12<\\/Type>\n <\\/Add>\n \n 16833612eebc283ce2fe3c447fb53eff<\\/ServerId>\n 0<\\/ParentId>\n Drafts<\\/DisplayName>\n 3<\\/Type>\n <\\/Add>\n \n ea8bb554b4246f8a48ccab88b653da54<\\/ServerId>\n 0<\\/ParentId>\n Sent<\\/DisplayName>\n 5<\\/Type>\n <\\/Add>\n \n 715ed9ea29b8a5377a69c1f758037c65<\\/ServerId>\n 0<\\/ParentId>\n Spam<\\/DisplayName>\n 12<\\/Type>\n <\\/Add>\n \n 22b8cf366c106d7c8253f4d390f697b8<\\/ServerId>\n 0<\\/ParentId>\n Trash<\\/DisplayName>\n 4<\\/Type>\n <\\/Add>\n \n DAV:task:f8a6ef015ca8eb311f00bc145523363da<\\/ServerId>\n 0<\\/ParentId>\n Calendar<\\/DisplayName>\n 7<\\/Type>\n <\\/Add>\n <\\/Changes>\n<\\/FolderSync>\n") + assert_eq!(.component, "syncroton") + assert_eq!(.user, "admin@kolab.local") assert!(is_timestamp(.timestamp)) - name: "IMAP SASL" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_imap-8ff9c4bdd-2h6sv_d59374a6-e466-446e-bcfa-dd0c0d089acb/imap/0.log container_id: containerd://eddf8182744573cbbba9bc7a44ddb80e757f2c45476ad5953a3a8b9ac64694cb container_image: localhost:5000/imap:latest container_image_id: localhost:5000/imap@sha256:17a7e91e1b657ef56d6805d6ea6daf57a5ef23ccfb6d40a7084ea58a01259bcc container_name: imap pod_name: imap-8ff9c4bdd-2h6sv pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/imap-8ff9c4bdd pod_uid: d59374a6-e466-446e-bcfa-dd0c0d089acb - message: 'saslauthd[12] :auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]' + log: 'saslauthd[12] :auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:17:16.927434664Z" outputs: - extract_from: "parse_imap" conditions: - type: "vrl" source: | - assert_eq!(.message, "auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]") + assert_eq!(.log, "auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]") assert!(is_timestamp(.timestamp)) - name: "IMAP Cyrus" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_imap-8ff9c4bdd-2h6sv_d59374a6-e466-446e-bcfa-dd0c0d089acb/imap/0.log container_id: containerd://eddf8182744573cbbba9bc7a44ddb80e757f2c45476ad5953a3a8b9ac64694cb container_image: localhost:5000/imap:latest container_image_id: localhost:5000/imap@sha256:17a7e91e1b657ef56d6805d6ea6daf57a5ef23ccfb6d40a7084ea58a01259bcc container_name: imap pod_name: imap-8ff9c4bdd-2h6sv pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/imap-8ff9c4bdd pod_uid: d59374a6-e466-446e-bcfa-dd0c0d089acb - message: 'cyrus-imapd/imap[13064]: USAGE admin@kolab.local user: 0.017951 sys: 0.011378' + log: 'cyrus-imapd/imap[13064]: USAGE admin@kolab.local user: 0.017951 sys: 0.011378' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:15:24.033133869Z" outputs: - extract_from: "parse_imap" conditions: - type: "vrl" source: | - assert_eq!(.message, "USAGE admin@kolab.local user: 0.017951 sys: 0.011378") + assert_eq!(.log, "USAGE admin@kolab.local user: 0.017951 sys: 0.011378") assert!(is_timestamp(.timestamp)) - name: "Mariadb warning" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_mariadb-6bcf7d9595-xg52b_b4836388-ed0e-4cca-81ee-97827274c290/mariadb/0.log container_id: containerd://c01aa737ade76451d5d7ed72e3605394442f77511e3defd5d1afe680e8662f84 container_image: localhost:5000/mariadb:latest container_image_id: localhost:5000/mariadb@sha256:249bbb724b9e9c687fde22ea3ef48c6a2bd0f8aa247a3a342a6b4598e5fb627b container_name: mariadb pod_name: mariadb-6bcf7d9595-xg52b pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/mariadb-6bcf7d9595 pod_uid: b4836388-ed0e-4cca-81ee-97827274c290 - message: "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)" + log: "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)" source_type: kubernetes_logs stream: stdout timestamp: "2024-08-19T10:57:36.696731329Z" outputs: - - extract_from: "apps._unmatched" + - extract_from: "parse_unmatched" conditions: - type: "vrl" source: | - assert_eq!(.message, "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)") + assert_eq!(.log, "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)") assert!(is_timestamp(.timestamp)) - name: "Proxy warning" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_proxy-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/proxy/0.log container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a container_image: localhost:5000/proxy:latest container_image_id: localhost:5000/proxy@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68 container_name: proxy pod_name: proxy-84bb89799b-rflm6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/proxy-84bb89799b pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3 - message: '2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443' + log: '2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T10:57:25.026161935Z" outputs: - extract_from: "parse_proxy" conditions: - type: "vrl" source: | - assert_eq!(.message, "2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443") + assert_eq!(.log, "2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443") assert!(is_timestamp(.timestamp)) diff --git a/docker/vector/rootfs/config/transforms.yaml b/docker/vector/rootfs/config/transforms.yaml index 6f7b992a..1010d614 100644 --- a/docker/vector/rootfs/config/transforms.yaml +++ b/docker/vector/rootfs/config/transforms.yaml @@ -1,58 +1,58 @@ transforms: apps: type: route inputs: - input reroute_unmatched: true route: roundcube: contains!(.pod_name, "roundcube") proxy: contains!(.pod_name, "proxy") kolab: contains!(.pod_name, "kolab") postfix: contains!(.pod_name, "postfix") imap: contains!(.pod_name, "imap") parse_kolab: type: remap inputs: - apps.kolab source: | - structured = parse_logfmt(.message) ?? {} + structured = parse_logfmt(.log) ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_roundcube: type: remap inputs: - apps.roundcube source: | - structured = parse_apache_log(.message, format: "common") ?? parse_logfmt(.message) ?? {} + structured = parse_apache_log(.log, format: "common") ?? parse_logfmt(.log) ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_imap: type: remap inputs: - apps.imap source: | - structured = parse_regex(.message, pattern: r'^(?.*)\[(?[0-9]+)\]( )?: *(?.*)$') ?? {} + structured = parse_regex(.log, pattern: r'^(?.*)\[(?[0-9]+)\]( )?: *(?.*)$') ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_proxy: type: remap inputs: - apps.proxy source: | - structured = parse_regex(.message, pattern: r'^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?