diff --git a/docker/imap/Dockerfile b/docker/imap/Dockerfile index ad48780c..030c2868 100644 --- a/docker/imap/Dockerfile +++ b/docker/imap/Dockerfile @@ -1,85 +1,85 @@ FROM apheleia/almalinux9 WORKDIR /root/ RUN dnf -y install \ --setopt 'tsflags=nodocs' \ git \ autoconf \ automake \ bison \ cyrus-sasl-devel \ flex \ gcc \ gcc-c++ \ gperf \ jansson-devel \ libbsd-devel \ libtool \ libicu-devel \ libuuid-devel \ openssl-devel \ pkgconfig \ sqlite-devel \ brotli-devel \ libzstd-devel \ libical-devel \ libxml2-devel \ libnghttp2-devel \ shapelib \ zlib-devel \ pcre-devel \ perl-devel \ cyrus-sasl \ cyrus-sasl-plain \ perl-Cyrus \ + busybox \ rsync && \ dnf clean all ARG GIT_REF=dev/kolab-3.6 ARG GIT_REMOTE=https://git.kolab.org/source/cyrus-imapd ADD build.sh /build.sh RUN /build.sh COPY /rootfs / VOLUME [ "/var/spool/imap" ] VOLUME [ "/var/lib/imap" ] RUN id default || (groupadd -g 1001 default && useradd -d /opt/app-root/ -u 1001 -g 1001 default) RUN mkdir -p /opt/app-root/src RUN PATHS=(/run /run/saslauthd /var/run /var/lib/imap /var/spool/imap /etc/pki/cyrus-imapd /opt/app-root) && \ mkdir -p ${PATHS[@]} && \ chmod 777 ${PATHS[@]} && \ chown -R 1001:0 ${PATHS[@]} && \ chmod -R g=u ${PATHS[@]} RUN touch /etc/pki/cyrus-imapd/cyrus-imapd.bundle.pem RUN PATHS=(/etc /etc/passwd /etc/saslauthd.conf /etc/cyrus.conf /etc/imapd.conf /etc/pki/cyrus-imapd/cyrus-imapd.bundle.pem) && \ chown 1001:0 ${PATHS[@]} && \ chmod g=u ${PATHS[@]} -USER 1001 WORKDIR /opt/app-root/src ENV SERVICES_PORT=8000 ENV IMAPD_CONF=/etc/imapd.conf ENV CYRUS_CONF=/etc/cyrus.conf ENV SERVERLIST=imap-backend ENV MUPDATE=imap-mupdate ENV SERVERNAME=imap-backend ENV MAXLOGINS_PER_USER=50 # Seems to be required on ppc64le only? Not sure why ENV LD_LIBRARY_PATH=/usr/lib/ # ENV TLS_SERVER_CA_FILE # ENV APP_SERVICES_DOMAIN # ENV ROLE # ENV WITH_TLS # ENV WITH_TAGS # ENV SYNC_HOST CMD ["/init.sh"] EXPOSE 11143/tcp 11993/tcp 11080/tcp 11443/tcp 11024/tcp 4190/tcp diff --git a/docker/imap/rootfs/init.sh b/docker/imap/rootfs/init.sh index 5c91ca2a..1f703c6f 100755 --- a/docker/imap/rootfs/init.sh +++ b/docker/imap/rootfs/init.sh @@ -1,128 +1,135 @@ #!/bin/bash set -e +set -x sed -i -r \ -e "s|IMAP_ADMIN_LOGIN|$IMAP_ADMIN_LOGIN|g" \ -e "s|IMAP_ADMIN_PASSWORD|$IMAP_ADMIN_PASSWORD|g" \ -e "s|MUPDATE_SERVER|$MUPDATE|g" \ -e "s|SERVERLIST|$SERVERLIST|g" \ -e "s|SERVERNAME|$SERVERNAME|g" \ -e "s|MAXLOGINS_PER_USER|$MAXLOGINS_PER_USER|g" \ -e "s|TLS_SERVER_CA_FILE|$TLS_SERVER_CA_FILE|g" \ $IMAPD_CONF sed -i -r \ -e "s|APP_SERVICES_DOMAIN|$APP_SERVICES_DOMAIN|g" \ -e "s|SERVICES_PORT|$SERVICES_PORT|g" \ /etc/saslauthd.conf if [[ "$CYRUS_CONF" != "/etc/cyrus.conf" ]]; then cp "$CYRUS_CONF" /etc/cyrus.conf fi if [[ "$IMAPD_CONF" != "/etc/imapd.conf" ]]; then cp "$IMAPD_CONF" /etc/imapd.conf fi -mkdir -p /var/lib/imap/socket -mkdir -p /var/lib/imap/db - if [[ "$WITH_TLS" == "true" ]]; then if [[ -f ${SSL_CERTIFICATE} ]]; then cat ${SSL_CERTIFICATE} ${SSL_CERTIFICATE_FULLCHAIN} ${SSL_CERTIFICATE_KEY} > /etc/pki/cyrus-imapd/cyrus-imapd.bundle.pem fi sed -i \ -e "s|# WITH_TLS ||g" \ /etc/imapd.conf sed -i \ -e "s|# WITH_TLS ||g" \ /etc/cyrus.conf fi if [[ "$WITH_TAGS" == "true" ]]; then sed -i \ -e "s|# WITH_TAGS ||g" \ /etc/imapd.conf else sed -i \ -e "s|# WITHOUT_TAGS ||g" \ /etc/imapd.conf fi if [[ "$SYNC_HOST" != "" ]]; then sed -i \ -e "s|# WITH_SYNC_TARGET ||g" \ -e "s|SYNC_HOST|$SYNC_HOST|g" \ /etc/imapd.conf sed -i \ -e "s|# WITH_SYNC_TARGET ||g" \ /etc/cyrus.conf fi if [[ "$ROLE" == "frontend" ]]; then sed -i \ -e "s|# WITH_MUPDATE ||g" \ -e "s|# ROLE_FRONTEND ||g" \ /etc/imapd.conf sed -i \ -e "s|# ROLE_FRONTEND ||g" \ /etc/cyrus.conf if [[ "$WITH_TLS" == "true" ]]; then sed -i \ -e "s|# ROLE_FRONTEND_WITH_TLS ||g" \ /etc/cyrus.conf fi elif [[ "$ROLE" == "backend" ]]; then sed -i \ -e "s|# WITH_MUPDATE ||g" \ -e "s|# ROLE_BACKEND ||g" \ /etc/imapd.conf sed -i \ -e "s|# WITH_MUPDATE ||g" \ -e "s|# ROLE_BACKEND ||g" \ /etc/cyrus.conf if [[ "$WITH_TLS" == "true" ]]; then sed -i \ -e "s|# ROLE_BACKEND_WITH_TLS ||g" \ /etc/cyrus.conf fi else sed -i \ -e "s|# ROLE_BACKEND ||g" \ /etc/imapd.conf sed -i \ -e "s|# ROLE_BACKEND ||g" \ /etc/cyrus.conf if [[ "$WITH_TLS" == "true" ]]; then sed -i \ -e "s|# ROLE_BACKEND_WITH_TLS ||g" \ /etc/cyrus.conf fi fi -/usr/sbin/saslauthd -m /run/saslauthd -a httpform -d & # Can't run as user because of /dev/ permissions so far. # Cyrus imap only logs to /dev/log, no way around it it seems. -# sudo rsyslogd - +busybox syslogd -n -O- & # Cyrus needs an entry in /etc/passwd. The alternative would perhaps be the nss_wrapper. # https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines # FIXME: This probably currently just works because we make /etc/ writable, which I suppose we shouldn't. -ID=$(id -u) -GID=$(id -g) +ID=$(id -u default) +GID=$(id -g default) echo "$ID:x:$ID:$GID::/opt/app-root/:/bin/bash" > /etc/passwd +runuser -u "$ID" -- /usr/sbin/saslauthd -m /run/saslauthd -a httpform -d & + +chown -R "$ID:$GID" /var/spool/imap/ +chown -R "$ID:$GID" /var/lib/imap/ + +runuser -u "$ID" -- mkdir -p /var/lib/imap/socket +runuser -u "$ID" -- mkdir -p /var/lib/imap/db + +export CYRUS_USER="$ID" +export CYRUS_VERBOSE=1 +# This will print a warning about a missing /var/lib/imap/db/skipstamp, but will still validate the config +runuser -u "$ID" -- cyr_info conf-lint + if [[ "$1" == "validate" ]]; then - # This will print a warning about a missing /var/lib/imap/db/skipstamp, but will still validate the config - cyr_info conf-lint echo "Config validated" else - cyr_info conf-lint - exec env CYRUS_VERBOSE=1 CYRUS_USER="$ID" /usr/libexec/master -D -p /var/run/master.pid + echo "Starting cyrus" + runuser -u "$ID" -- /usr/libexec/master -D -p /var/run/master.pid fi diff --git a/docker/vector/rootfs/config/tests.yaml b/docker/vector/rootfs/config/tests.yaml index e452d6ad..ca86f63c 100644 --- a/docker/vector/rootfs/config/tests.yaml +++ b/docker/vector/rootfs/config/tests.yaml @@ -1,319 +1,319 @@ tests: - name: "Roundcube logfmt output" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-78bd5fccb-lssj6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-78bd5fccb pod_uid: cc681420-f781-4332-b966-73812b372801 log: name=kolabfiles component=chwala session=a410dfeb user=admin@kolab.local log="Refreshing the access token" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:18:49.006951369Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | assert_eq!(.component, "chwala") assert_eq!(.user, "admin@kolab.local") assert_eq!(.log, "Refreshing the access token") assert!(is_timestamp(.timestamp)) - name: "Roundcube php warning" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-78bd5fccb-lssj6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-78bd5fccb pod_uid: cc681420-f781-4332-b966-73812b372801 log: 'PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:19:06.317485633Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | assert_eq!(.log, "PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667") assert!(is_timestamp(.timestamp)) - name: "Roundcube generic message" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-74d95fdbf5-9vq5n_7625ef26-43e4-4398-956f-c8b9f10c38e8/roundcube/0.log container_id: containerd://8aaa3192e31ef776307f7ea96d3c8fa71a15494dd22a400ae7d74f3e221fc33b container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-74d95fdbf5-9vq5n pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-74d95fdbf5 pod_uid: 7625ef26-43e4-4398-956f-c8b9f10c38e8 log: "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-18T09:18:38.276041789Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | assert_eq!(.log, "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)") assert!(is_timestamp(.timestamp)) - name: "Roundcube apache" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-74d95fdbf5-9vq5n_7625ef26-43e4-4398-956f-c8b9f10c38e8/roundcube/0.log container_id: containerd://8aaa3192e31ef776307f7ea96d3c8fa71a15494dd22a400ae7d74f3e221fc33b container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-74d95fdbf5-9vq5n pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-74d95fdbf5 pod_uid: 7625ef26-43e4-4398-956f-c8b9f10c38e8 log: "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-18T09:18:38.276041789Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | assert_eq!(.log, "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"") assert!(is_timestamp(.timestamp)) - name: "Syncroton" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a container_image: localhost:5000/roundcube:latest container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879 container_name: roundcube pod_name: roundcube-78bd5fccb-lssj6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/roundcube-78bd5fccb pod_uid: cc681420-f781-4332-b966-73812b372801 log: name=console component=syncroton cmd=FolderSync device=v140Device type=iphone user=admin@kolab.local log="Syncroton_Server::_handlePost::186 xml response (0):\n<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE AirSync PUBLIC \"-\/\/AIRSYNC\/\/DTD AirSync\/\/EN\" \"http:\/\/www.microsoft.com\/\">\n<FolderSync xmlns=\"uri:FolderHierarchy\" xmlns:Syncroton=\"uri:Syncroton\" xmlns:Internal=\"uri:Internal\">\n <Status>1<\/Status>\n <SyncKey>1<\/SyncKey>\n <Changes>\n <Count>9<\/Count>\n <Add>\n <ServerId>DAV:event:f8a6ef015ca8eb311f00bc145523363da<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Calendar<\/DisplayName>\n <Type>8<\/Type>\n <\/Add>\n <Add>\n <ServerId>DAV:contact:f027577d9ee8a8fafaf314f1a5db41a81<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Addressbook<\/DisplayName>\n <Type>9<\/Type>\n <\/Add>\n <Add>\n <ServerId>38b950ebd62cd9a66929c89615d0fc04<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>INBOX<\/DisplayName>\n <Type>2<\/Type>\n <\/Add>\n <Add>\n <ServerId>1bb8c55fe84d52c6968db2571f7dc124<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Archive<\/DisplayName>\n <Type>12<\/Type>\n <\/Add>\n <Add>\n <ServerId>16833612eebc283ce2fe3c447fb53eff<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Drafts<\/DisplayName>\n <Type>3<\/Type>\n <\/Add>\n <Add>\n <ServerId>ea8bb554b4246f8a48ccab88b653da54<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Sent<\/DisplayName>\n <Type>5<\/Type>\n <\/Add>\n <Add>\n <ServerId>715ed9ea29b8a5377a69c1f758037c65<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Spam<\/DisplayName>\n <Type>12<\/Type>\n <\/Add>\n <Add>\n <ServerId>22b8cf366c106d7c8253f4d390f697b8<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Trash<\/DisplayName>\n <Type>4<\/Type>\n <\/Add>\n <Add>\n <ServerId>DAV:task:f8a6ef015ca8eb311f00bc145523363da<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Calendar<\/DisplayName>\n <Type>7<\/Type>\n <\/Add>\n <\/Changes>\n<\/FolderSync>\n" source_type: kubernetes_logs stream: stderr timestamp: "2024-08-20T13:19:46.879013117Z" outputs: - extract_from: "parse_roundcube" conditions: - type: "vrl" source: | assert_eq!(.log, "Syncroton_Server::_handlePost::186 xml response (0):\n<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE AirSync PUBLIC \"-\\/\\/AIRSYNC\\/\\/DTD AirSync\\/\\/EN\" \"http:\\/\\/www.microsoft.com\\/\">\n<FolderSync xmlns=\"uri:FolderHierarchy\" xmlns:Syncroton=\"uri:Syncroton\" xmlns:Internal=\"uri:Internal\">\n <Status>1<\\/Status>\n <SyncKey>1<\\/SyncKey>\n <Changes>\n <Count>9<\\/Count>\n <Add>\n <ServerId>DAV:event:f8a6ef015ca8eb311f00bc145523363da<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Calendar<\\/DisplayName>\n <Type>8<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>DAV:contact:f027577d9ee8a8fafaf314f1a5db41a81<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Addressbook<\\/DisplayName>\n <Type>9<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>38b950ebd62cd9a66929c89615d0fc04<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>INBOX<\\/DisplayName>\n <Type>2<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>1bb8c55fe84d52c6968db2571f7dc124<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Archive<\\/DisplayName>\n <Type>12<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>16833612eebc283ce2fe3c447fb53eff<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Drafts<\\/DisplayName>\n <Type>3<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>ea8bb554b4246f8a48ccab88b653da54<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Sent<\\/DisplayName>\n <Type>5<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>715ed9ea29b8a5377a69c1f758037c65<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Spam<\\/DisplayName>\n <Type>12<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>22b8cf366c106d7c8253f4d390f697b8<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Trash<\\/DisplayName>\n <Type>4<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>DAV:task:f8a6ef015ca8eb311f00bc145523363da<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Calendar<\\/DisplayName>\n <Type>7<\\/Type>\n <\\/Add>\n <\\/Changes>\n<\\/FolderSync>\n") assert_eq!(.component, "syncroton") assert_eq!(.user, "admin@kolab.local") assert!(is_timestamp(.timestamp)) - name: "IMAP SASL" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_imap-8ff9c4bdd-2h6sv_d59374a6-e466-446e-bcfa-dd0c0d089acb/imap/0.log container_id: containerd://eddf8182744573cbbba9bc7a44ddb80e757f2c45476ad5953a3a8b9ac64694cb container_image: localhost:5000/imap:latest container_image_id: localhost:5000/imap@sha256:17a7e91e1b657ef56d6805d6ea6daf57a5ef23ccfb6d40a7084ea58a01259bcc container_name: imap pod_name: imap-8ff9c4bdd-2h6sv pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/imap-8ff9c4bdd pod_uid: d59374a6-e466-446e-bcfa-dd0c0d089acb - log: 'saslauthd[12] :auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]' + log: 'Mar 11 19:00:42 imap-f99cb6ddc-4gplf auth.debug saslauthd[12] :auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:17:16.927434664Z" outputs: - extract_from: "parse_imap" conditions: - type: "vrl" source: | assert_eq!(.log, "auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]") assert!(is_timestamp(.timestamp)) - name: "IMAP Cyrus" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_imap-8ff9c4bdd-2h6sv_d59374a6-e466-446e-bcfa-dd0c0d089acb/imap/0.log container_id: containerd://eddf8182744573cbbba9bc7a44ddb80e757f2c45476ad5953a3a8b9ac64694cb container_image: localhost:5000/imap:latest container_image_id: localhost:5000/imap@sha256:17a7e91e1b657ef56d6805d6ea6daf57a5ef23ccfb6d40a7084ea58a01259bcc container_name: imap pod_name: imap-8ff9c4bdd-2h6sv pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/imap-8ff9c4bdd pod_uid: d59374a6-e466-446e-bcfa-dd0c0d089acb - log: 'cyrus-imapd/imap[13064]: USAGE admin@kolab.local user: 0.017951 sys: 0.011378' + log: 'Mar 11 19:00:42 imap-f99cb6ddc-4gplf local6.info cyrus-imapd/imap[13064]: USAGE admin@kolab.local user: 0.017951 sys: 0.011378' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T20:15:24.033133869Z" outputs: - extract_from: "parse_imap" conditions: - type: "vrl" source: | assert_eq!(.log, "USAGE admin@kolab.local user: 0.017951 sys: 0.011378") assert!(is_timestamp(.timestamp)) - name: "Mariadb warning" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_mariadb-6bcf7d9595-xg52b_b4836388-ed0e-4cca-81ee-97827274c290/mariadb/0.log container_id: containerd://c01aa737ade76451d5d7ed72e3605394442f77511e3defd5d1afe680e8662f84 container_image: localhost:5000/mariadb:latest container_image_id: localhost:5000/mariadb@sha256:249bbb724b9e9c687fde22ea3ef48c6a2bd0f8aa247a3a342a6b4598e5fb627b container_name: mariadb pod_name: mariadb-6bcf7d9595-xg52b pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/mariadb-6bcf7d9595 pod_uid: b4836388-ed0e-4cca-81ee-97827274c290 log: "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)" source_type: kubernetes_logs stream: stdout timestamp: "2024-08-19T10:57:36.696731329Z" outputs: - extract_from: "parse_unmatched" conditions: - type: "vrl" source: | assert_eq!(.log, "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)") assert!(is_timestamp(.timestamp)) - name: "Proxy warning" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_proxy-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/proxy/0.log container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a container_image: localhost:5000/proxy:latest container_image_id: localhost:5000/proxy@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68 container_name: proxy pod_name: proxy-84bb89799b-rflm6 pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/proxy-84bb89799b pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3 log: '2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T10:57:25.026161935Z" outputs: - extract_from: "parse_proxy" conditions: - type: "vrl" source: | assert_eq!(.log, "2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443") assert!(is_timestamp(.timestamp)) - name: "Postfix" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_postfix-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/postfix/0.log container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a container_image: localhost:5000/postfix:latest container_image_id: localhost:5000/postfix@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68 container_name: postfix pod_name: postfix-5fbfb788c-mjjxh pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/postfix-84bb89799b pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3 log: 'EB6AA233B6D2: to=<admin@single.k3s.klab.cc>, relay=imap[10.43.9.216]:11024, delay=0.03, delays=0.01/0/0/0.02, dsn=2.1.5, status=sent (250 2.1.5 Success SESSIONID=<cyrus-imapd-1741268864-161809-1-11604045405236919686>)' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T10:57:25.026161935Z" outputs: - extract_from: "parse_postfix" conditions: - type: "vrl" source: | assert_eq!(.log, "EB6AA233B6D2: to=<admin@single.k3s.klab.cc>, relay=imap[10.43.9.216]:11024, delay=0.03, delays=0.01/0/0/0.02, dsn=2.1.5, status=sent (250 2.1.5 Success SESSIONID=<cyrus-imapd-1741268864-161809-1-11604045405236919686>)") assert!(is_timestamp(.timestamp)) - name: "Postfix smtp queued" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_postfix-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/postfix/0.log container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a container_image: localhost:5000/postfix:latest container_image_id: localhost:5000/postfix@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68 container_name: postfix pod_name: postfix-5fbfb788c-mjjxh pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/postfix-84bb89799b pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3 log: 'Mar 06 13:20:09 postfix postfix/smtp[68356]: AEAFC233B6D2: to=<test2@single.k3s.klab.cc>, relay=127.0.0.1[127.0.0.1]:13026, delay=7.7, delays=2.4/0.04/0.01/5.2, dsn=2.0.0, status=sent (250 2.0.0 from MTA(smtp:[127.0.0.1]:13025): 250 2.0.0 Ok: queued as 231D4238BAE4)' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T10:57:25.026161935Z" outputs: - extract_from: "parse_postfix" conditions: - type: "vrl" source: | assert_eq!(.log, "AEAFC233B6D2: to=<test2@single.k3s.klab.cc>, relay=127.0.0.1[127.0.0.1]:13026, delay=7.7, delays=2.4/0.04/0.01/5.2, dsn=2.0.0, status=sent (250 2.0.0 from MTA(smtp:[127.0.0.1]:13025): 250 2.0.0 Ok: queued as 231D4238BAE4)") assert!(is_timestamp(.timestamp)) assert_eq!(.process, "postfix/smtp[68356]") - name: "Postfix saslauthd" inputs: - insert_at: "apps" type: "log" log_fields: file: /var/log/pods/kolab_postfix-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/postfix/0.log container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a container_image: localhost:5000/postfix:latest container_image_id: localhost:5000/postfix@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68 container_name: postfix pod_name: postfix-5fbfb788c-mjjxh pod_namespace: kolab pod_node_name: dws07.kolabsys.com pod_owner: ReplicaSet/postfix-84bb89799b pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3 log: 'saslauthd[471] :auth success: [user=admin] [service=smtp] [realm=single.k3s.klab.cc] [mech=httpform]' source_type: kubernetes_logs stream: stderr timestamp: "2024-08-19T10:57:25.026161935Z" outputs: - extract_from: "parse_postfix" conditions: - type: "vrl" source: | assert_eq!(.log, "saslauthd[471] :auth success: [user=admin] [service=smtp] [realm=single.k3s.klab.cc] [mech=httpform]") assert!(is_timestamp(.timestamp)) diff --git a/docker/vector/rootfs/config/transforms.yaml b/docker/vector/rootfs/config/transforms.yaml index a68e38ab..29f94d54 100644 --- a/docker/vector/rootfs/config/transforms.yaml +++ b/docker/vector/rootfs/config/transforms.yaml @@ -1,69 +1,69 @@ transforms: apps: type: route inputs: - input reroute_unmatched: true route: roundcube: contains!(.pod_name, "roundcube") proxy: contains!(.pod_name, "proxy") kolab: contains!(.pod_name, "kolab") postfix: contains!(.pod_name, "postfix") imap: contains!(.pod_name, "imap") parse_kolab: type: remap inputs: - apps.kolab source: | structured = parse_logfmt(.log) ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_roundcube: type: remap inputs: - apps.roundcube source: | structured = parse_apache_log(.log, format: "common") ?? parse_logfmt(.log) ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_imap: type: remap inputs: - apps.imap source: | # Drop noisy saslauthd debug messages if contains(string!(.log), "accept lock") { abort } - structured = parse_regex(.log, pattern: r'^(?<program>.*)\[(?<pid>[0-9]+)\]( )?: *(?<log>.*)$') ?? {} + structured = parse_regex(.log, pattern: r'^(?<time>[^ ]+ [0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]) (?<host>[^ ]+) (?<facility>[^ ]+) (?<program>.*)\[(?<pid>[0-9]+)\]( )?: *(?<log>.*)$') ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_proxy: type: remap inputs: - apps.proxy source: | structured = parse_regex(.log, pattern: r'^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")') ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_postfix: type: remap inputs: - apps.postfix source: | if .container_name == "metricsexporter" { abort } # Drop noisy saslauthd debug messages if contains(string!(.log), "accept lock") { abort } structured = parse_regex(.log, pattern: r'^(?<time>[^ ]+ [0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]) (?<host>[^ ]+) (?<process>[^:]+): (?<log>((?<key>[^ :]+)[ :])? ?((to|from)=<(?<address>[^>]+)>)?.*)$') ?? {} . = merge(., structured) .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now() parse_unmatched: type: remap inputs: - apps._unmatched source: | .timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()