diff --git a/docker/imap/Dockerfile b/docker/imap/Dockerfile
index ad48780c..030c2868 100644
--- a/docker/imap/Dockerfile
+++ b/docker/imap/Dockerfile
@@ -1,85 +1,85 @@
FROM apheleia/almalinux9
WORKDIR /root/
RUN dnf -y install \
--setopt 'tsflags=nodocs' \
git \
autoconf \
automake \
bison \
cyrus-sasl-devel \
flex \
gcc \
gcc-c++ \
gperf \
jansson-devel \
libbsd-devel \
libtool \
libicu-devel \
libuuid-devel \
openssl-devel \
pkgconfig \
sqlite-devel \
brotli-devel \
libzstd-devel \
libical-devel \
libxml2-devel \
libnghttp2-devel \
shapelib \
zlib-devel \
pcre-devel \
perl-devel \
cyrus-sasl \
cyrus-sasl-plain \
perl-Cyrus \
+ busybox \
rsync && \
dnf clean all
ARG GIT_REF=dev/kolab-3.6
ARG GIT_REMOTE=https://git.kolab.org/source/cyrus-imapd
ADD build.sh /build.sh
RUN /build.sh
COPY /rootfs /
VOLUME [ "/var/spool/imap" ]
VOLUME [ "/var/lib/imap" ]
RUN id default || (groupadd -g 1001 default && useradd -d /opt/app-root/ -u 1001 -g 1001 default)
RUN mkdir -p /opt/app-root/src
RUN PATHS=(/run /run/saslauthd /var/run /var/lib/imap /var/spool/imap /etc/pki/cyrus-imapd /opt/app-root) && \
mkdir -p ${PATHS[@]} && \
chmod 777 ${PATHS[@]} && \
chown -R 1001:0 ${PATHS[@]} && \
chmod -R g=u ${PATHS[@]}
RUN touch /etc/pki/cyrus-imapd/cyrus-imapd.bundle.pem
RUN PATHS=(/etc /etc/passwd /etc/saslauthd.conf /etc/cyrus.conf /etc/imapd.conf /etc/pki/cyrus-imapd/cyrus-imapd.bundle.pem) && \
chown 1001:0 ${PATHS[@]} && \
chmod g=u ${PATHS[@]}
-USER 1001
WORKDIR /opt/app-root/src
ENV SERVICES_PORT=8000
ENV IMAPD_CONF=/etc/imapd.conf
ENV CYRUS_CONF=/etc/cyrus.conf
ENV SERVERLIST=imap-backend
ENV MUPDATE=imap-mupdate
ENV SERVERNAME=imap-backend
ENV MAXLOGINS_PER_USER=50
# Seems to be required on ppc64le only? Not sure why
ENV LD_LIBRARY_PATH=/usr/lib/
# ENV TLS_SERVER_CA_FILE
# ENV APP_SERVICES_DOMAIN
# ENV ROLE
# ENV WITH_TLS
# ENV WITH_TAGS
# ENV SYNC_HOST
CMD ["/init.sh"]
EXPOSE 11143/tcp 11993/tcp 11080/tcp 11443/tcp 11024/tcp 4190/tcp
diff --git a/docker/imap/rootfs/init.sh b/docker/imap/rootfs/init.sh
index 5c91ca2a..1f703c6f 100755
--- a/docker/imap/rootfs/init.sh
+++ b/docker/imap/rootfs/init.sh
@@ -1,128 +1,135 @@
#!/bin/bash
set -e
+set -x
sed -i -r \
-e "s|IMAP_ADMIN_LOGIN|$IMAP_ADMIN_LOGIN|g" \
-e "s|IMAP_ADMIN_PASSWORD|$IMAP_ADMIN_PASSWORD|g" \
-e "s|MUPDATE_SERVER|$MUPDATE|g" \
-e "s|SERVERLIST|$SERVERLIST|g" \
-e "s|SERVERNAME|$SERVERNAME|g" \
-e "s|MAXLOGINS_PER_USER|$MAXLOGINS_PER_USER|g" \
-e "s|TLS_SERVER_CA_FILE|$TLS_SERVER_CA_FILE|g" \
$IMAPD_CONF
sed -i -r \
-e "s|APP_SERVICES_DOMAIN|$APP_SERVICES_DOMAIN|g" \
-e "s|SERVICES_PORT|$SERVICES_PORT|g" \
/etc/saslauthd.conf
if [[ "$CYRUS_CONF" != "/etc/cyrus.conf" ]]; then
cp "$CYRUS_CONF" /etc/cyrus.conf
fi
if [[ "$IMAPD_CONF" != "/etc/imapd.conf" ]]; then
cp "$IMAPD_CONF" /etc/imapd.conf
fi
-mkdir -p /var/lib/imap/socket
-mkdir -p /var/lib/imap/db
-
if [[ "$WITH_TLS" == "true" ]]; then
if [[ -f ${SSL_CERTIFICATE} ]]; then
cat ${SSL_CERTIFICATE} ${SSL_CERTIFICATE_FULLCHAIN} ${SSL_CERTIFICATE_KEY} > /etc/pki/cyrus-imapd/cyrus-imapd.bundle.pem
fi
sed -i \
-e "s|# WITH_TLS ||g" \
/etc/imapd.conf
sed -i \
-e "s|# WITH_TLS ||g" \
/etc/cyrus.conf
fi
if [[ "$WITH_TAGS" == "true" ]]; then
sed -i \
-e "s|# WITH_TAGS ||g" \
/etc/imapd.conf
else
sed -i \
-e "s|# WITHOUT_TAGS ||g" \
/etc/imapd.conf
fi
if [[ "$SYNC_HOST" != "" ]]; then
sed -i \
-e "s|# WITH_SYNC_TARGET ||g" \
-e "s|SYNC_HOST|$SYNC_HOST|g" \
/etc/imapd.conf
sed -i \
-e "s|# WITH_SYNC_TARGET ||g" \
/etc/cyrus.conf
fi
if [[ "$ROLE" == "frontend" ]]; then
sed -i \
-e "s|# WITH_MUPDATE ||g" \
-e "s|# ROLE_FRONTEND ||g" \
/etc/imapd.conf
sed -i \
-e "s|# ROLE_FRONTEND ||g" \
/etc/cyrus.conf
if [[ "$WITH_TLS" == "true" ]]; then
sed -i \
-e "s|# ROLE_FRONTEND_WITH_TLS ||g" \
/etc/cyrus.conf
fi
elif [[ "$ROLE" == "backend" ]]; then
sed -i \
-e "s|# WITH_MUPDATE ||g" \
-e "s|# ROLE_BACKEND ||g" \
/etc/imapd.conf
sed -i \
-e "s|# WITH_MUPDATE ||g" \
-e "s|# ROLE_BACKEND ||g" \
/etc/cyrus.conf
if [[ "$WITH_TLS" == "true" ]]; then
sed -i \
-e "s|# ROLE_BACKEND_WITH_TLS ||g" \
/etc/cyrus.conf
fi
else
sed -i \
-e "s|# ROLE_BACKEND ||g" \
/etc/imapd.conf
sed -i \
-e "s|# ROLE_BACKEND ||g" \
/etc/cyrus.conf
if [[ "$WITH_TLS" == "true" ]]; then
sed -i \
-e "s|# ROLE_BACKEND_WITH_TLS ||g" \
/etc/cyrus.conf
fi
fi
-/usr/sbin/saslauthd -m /run/saslauthd -a httpform -d &
# Can't run as user because of /dev/ permissions so far.
# Cyrus imap only logs to /dev/log, no way around it it seems.
-# sudo rsyslogd
-
+busybox syslogd -n -O- &
# Cyrus needs an entry in /etc/passwd. The alternative would perhaps be the nss_wrapper.
# https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines
# FIXME: This probably currently just works because we make /etc/ writable, which I suppose we shouldn't.
-ID=$(id -u)
-GID=$(id -g)
+ID=$(id -u default)
+GID=$(id -g default)
echo "$ID:x:$ID:$GID::/opt/app-root/:/bin/bash" > /etc/passwd
+runuser -u "$ID" -- /usr/sbin/saslauthd -m /run/saslauthd -a httpform -d &
+
+chown -R "$ID:$GID" /var/spool/imap/
+chown -R "$ID:$GID" /var/lib/imap/
+
+runuser -u "$ID" -- mkdir -p /var/lib/imap/socket
+runuser -u "$ID" -- mkdir -p /var/lib/imap/db
+
+export CYRUS_USER="$ID"
+export CYRUS_VERBOSE=1
+# This will print a warning about a missing /var/lib/imap/db/skipstamp, but will still validate the config
+runuser -u "$ID" -- cyr_info conf-lint
+
if [[ "$1" == "validate" ]]; then
- # This will print a warning about a missing /var/lib/imap/db/skipstamp, but will still validate the config
- cyr_info conf-lint
echo "Config validated"
else
- cyr_info conf-lint
- exec env CYRUS_VERBOSE=1 CYRUS_USER="$ID" /usr/libexec/master -D -p /var/run/master.pid
+ echo "Starting cyrus"
+ runuser -u "$ID" -- /usr/libexec/master -D -p /var/run/master.pid
fi
diff --git a/docker/vector/rootfs/config/tests.yaml b/docker/vector/rootfs/config/tests.yaml
index e452d6ad..ca86f63c 100644
--- a/docker/vector/rootfs/config/tests.yaml
+++ b/docker/vector/rootfs/config/tests.yaml
@@ -1,319 +1,319 @@
tests:
- name: "Roundcube logfmt output"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log
container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a
container_image: localhost:5000/roundcube:latest
container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879
container_name: roundcube
pod_name: roundcube-78bd5fccb-lssj6
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/roundcube-78bd5fccb
pod_uid: cc681420-f781-4332-b966-73812b372801
log: name=kolabfiles component=chwala session=a410dfeb user=admin@kolab.local log="Refreshing the access token"
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T20:18:49.006951369Z"
outputs:
- extract_from: "parse_roundcube"
conditions:
- type: "vrl"
source: |
assert_eq!(.component, "chwala")
assert_eq!(.user, "admin@kolab.local")
assert_eq!(.log, "Refreshing the access token")
assert!(is_timestamp(.timestamp))
- name: "Roundcube php warning"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log
container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a
container_image: localhost:5000/roundcube:latest
container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879
container_name: roundcube
pod_name: roundcube-78bd5fccb-lssj6
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/roundcube-78bd5fccb
pod_uid: cc681420-f781-4332-b966-73812b372801
log: 'PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T20:19:06.317485633Z"
outputs:
- extract_from: "parse_roundcube"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "PHP message: PHP Warning: Array to string conversion in /opt/app-root/src/roundcubemail/vendor/pear/crypt_gpg/Crypt/GPG/Engine.php on line 1667")
assert!(is_timestamp(.timestamp))
- name: "Roundcube generic message"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_roundcube-74d95fdbf5-9vq5n_7625ef26-43e4-4398-956f-c8b9f10c38e8/roundcube/0.log
container_id: containerd://8aaa3192e31ef776307f7ea96d3c8fa71a15494dd22a400ae7d74f3e221fc33b
container_image: localhost:5000/roundcube:latest
container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879
container_name: roundcube
pod_name: roundcube-74d95fdbf5-9vq5n
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/roundcube-74d95fdbf5
pod_uid: 7625ef26-43e4-4398-956f-c8b9f10c38e8
log: "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)"
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-18T09:18:38.276041789Z"
outputs:
- extract_from: "parse_roundcube"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "+ for plugin in $(find plugins -mindepth 1 -maxdepth 1 -type d | sort)")
assert!(is_timestamp(.timestamp))
- name: "Roundcube apache"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_roundcube-74d95fdbf5-9vq5n_7625ef26-43e4-4398-956f-c8b9f10c38e8/roundcube/0.log
container_id: containerd://8aaa3192e31ef776307f7ea96d3c8fa71a15494dd22a400ae7d74f3e221fc33b
container_image: localhost:5000/roundcube:latest
container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879
container_name: roundcube
pod_name: roundcube-74d95fdbf5-9vq5n
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/roundcube-74d95fdbf5
pod_uid: 7625ef26-43e4-4398-956f-c8b9f10c38e8
log: "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\""
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-18T09:18:38.276041789Z"
outputs:
- extract_from: "parse_roundcube"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "10.42.0.238 - - [20/Aug/2024:09:28:05 +0000] \"POST /webmail/71NpOUihc2TMCKJy/?_task=mail&_action=refresh HTTP/1.0\" 200 193 \"https://kolab.local/webmail/71NpOUihc2TMCKJy/?_task=mail&_mbox=INBOX\" \"Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\"")
assert!(is_timestamp(.timestamp))
- name: "Syncroton"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_roundcube-78bd5fccb-lssj6_cc681420-f781-4332-b966-73812b372801/roundcube/0.log
container_id: containerd://e3a3059ac016578c967f4034831feff7fea79c5152f92f0cfb523e779fdd0a0a
container_image: localhost:5000/roundcube:latest
container_image_id: localhost:5000/roundcube@sha256:01b87d337ef0fe7e15943b38dc2b4023638ef0a06dabbc0ac387128bc744e879
container_name: roundcube
pod_name: roundcube-78bd5fccb-lssj6
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/roundcube-78bd5fccb
pod_uid: cc681420-f781-4332-b966-73812b372801
log: name=console component=syncroton cmd=FolderSync device=v140Device type=iphone user=admin@kolab.local log="Syncroton_Server::_handlePost::186 xml response (0):\n<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE AirSync PUBLIC \"-\/\/AIRSYNC\/\/DTD AirSync\/\/EN\" \"http:\/\/www.microsoft.com\/\">\n<FolderSync xmlns=\"uri:FolderHierarchy\" xmlns:Syncroton=\"uri:Syncroton\" xmlns:Internal=\"uri:Internal\">\n <Status>1<\/Status>\n <SyncKey>1<\/SyncKey>\n <Changes>\n <Count>9<\/Count>\n <Add>\n <ServerId>DAV:event:f8a6ef015ca8eb311f00bc145523363da<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Calendar<\/DisplayName>\n <Type>8<\/Type>\n <\/Add>\n <Add>\n <ServerId>DAV:contact:f027577d9ee8a8fafaf314f1a5db41a81<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Addressbook<\/DisplayName>\n <Type>9<\/Type>\n <\/Add>\n <Add>\n <ServerId>38b950ebd62cd9a66929c89615d0fc04<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>INBOX<\/DisplayName>\n <Type>2<\/Type>\n <\/Add>\n <Add>\n <ServerId>1bb8c55fe84d52c6968db2571f7dc124<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Archive<\/DisplayName>\n <Type>12<\/Type>\n <\/Add>\n <Add>\n <ServerId>16833612eebc283ce2fe3c447fb53eff<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Drafts<\/DisplayName>\n <Type>3<\/Type>\n <\/Add>\n <Add>\n <ServerId>ea8bb554b4246f8a48ccab88b653da54<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Sent<\/DisplayName>\n <Type>5<\/Type>\n <\/Add>\n <Add>\n <ServerId>715ed9ea29b8a5377a69c1f758037c65<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Spam<\/DisplayName>\n <Type>12<\/Type>\n <\/Add>\n <Add>\n <ServerId>22b8cf366c106d7c8253f4d390f697b8<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Trash<\/DisplayName>\n <Type>4<\/Type>\n <\/Add>\n <Add>\n <ServerId>DAV:task:f8a6ef015ca8eb311f00bc145523363da<\/ServerId>\n <ParentId>0<\/ParentId>\n <DisplayName>Calendar<\/DisplayName>\n <Type>7<\/Type>\n <\/Add>\n <\/Changes>\n<\/FolderSync>\n"
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-20T13:19:46.879013117Z"
outputs:
- extract_from: "parse_roundcube"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "Syncroton_Server::_handlePost::186 xml response (0):\n<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE AirSync PUBLIC \"-\\/\\/AIRSYNC\\/\\/DTD AirSync\\/\\/EN\" \"http:\\/\\/www.microsoft.com\\/\">\n<FolderSync xmlns=\"uri:FolderHierarchy\" xmlns:Syncroton=\"uri:Syncroton\" xmlns:Internal=\"uri:Internal\">\n <Status>1<\\/Status>\n <SyncKey>1<\\/SyncKey>\n <Changes>\n <Count>9<\\/Count>\n <Add>\n <ServerId>DAV:event:f8a6ef015ca8eb311f00bc145523363da<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Calendar<\\/DisplayName>\n <Type>8<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>DAV:contact:f027577d9ee8a8fafaf314f1a5db41a81<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Addressbook<\\/DisplayName>\n <Type>9<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>38b950ebd62cd9a66929c89615d0fc04<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>INBOX<\\/DisplayName>\n <Type>2<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>1bb8c55fe84d52c6968db2571f7dc124<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Archive<\\/DisplayName>\n <Type>12<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>16833612eebc283ce2fe3c447fb53eff<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Drafts<\\/DisplayName>\n <Type>3<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>ea8bb554b4246f8a48ccab88b653da54<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Sent<\\/DisplayName>\n <Type>5<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>715ed9ea29b8a5377a69c1f758037c65<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Spam<\\/DisplayName>\n <Type>12<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>22b8cf366c106d7c8253f4d390f697b8<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Trash<\\/DisplayName>\n <Type>4<\\/Type>\n <\\/Add>\n <Add>\n <ServerId>DAV:task:f8a6ef015ca8eb311f00bc145523363da<\\/ServerId>\n <ParentId>0<\\/ParentId>\n <DisplayName>Calendar<\\/DisplayName>\n <Type>7<\\/Type>\n <\\/Add>\n <\\/Changes>\n<\\/FolderSync>\n")
assert_eq!(.component, "syncroton")
assert_eq!(.user, "admin@kolab.local")
assert!(is_timestamp(.timestamp))
- name: "IMAP SASL"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_imap-8ff9c4bdd-2h6sv_d59374a6-e466-446e-bcfa-dd0c0d089acb/imap/0.log
container_id: containerd://eddf8182744573cbbba9bc7a44ddb80e757f2c45476ad5953a3a8b9ac64694cb
container_image: localhost:5000/imap:latest
container_image_id: localhost:5000/imap@sha256:17a7e91e1b657ef56d6805d6ea6daf57a5ef23ccfb6d40a7084ea58a01259bcc
container_name: imap
pod_name: imap-8ff9c4bdd-2h6sv
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/imap-8ff9c4bdd
pod_uid: d59374a6-e466-446e-bcfa-dd0c0d089acb
- log: 'saslauthd[12] :auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]'
+ log: 'Mar 11 19:00:42 imap-f99cb6ddc-4gplf auth.debug saslauthd[12] :auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T20:17:16.927434664Z"
outputs:
- extract_from: "parse_imap"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "auth success: [user=admin] [service=imap] [realm=kolab.local] [mech=httpform]")
assert!(is_timestamp(.timestamp))
- name: "IMAP Cyrus"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_imap-8ff9c4bdd-2h6sv_d59374a6-e466-446e-bcfa-dd0c0d089acb/imap/0.log
container_id: containerd://eddf8182744573cbbba9bc7a44ddb80e757f2c45476ad5953a3a8b9ac64694cb
container_image: localhost:5000/imap:latest
container_image_id: localhost:5000/imap@sha256:17a7e91e1b657ef56d6805d6ea6daf57a5ef23ccfb6d40a7084ea58a01259bcc
container_name: imap
pod_name: imap-8ff9c4bdd-2h6sv
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/imap-8ff9c4bdd
pod_uid: d59374a6-e466-446e-bcfa-dd0c0d089acb
- log: 'cyrus-imapd/imap[13064]: USAGE admin@kolab.local user: 0.017951 sys: 0.011378'
+ log: 'Mar 11 19:00:42 imap-f99cb6ddc-4gplf local6.info cyrus-imapd/imap[13064]: USAGE admin@kolab.local user: 0.017951 sys: 0.011378'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T20:15:24.033133869Z"
outputs:
- extract_from: "parse_imap"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "USAGE admin@kolab.local user: 0.017951 sys: 0.011378")
assert!(is_timestamp(.timestamp))
- name: "Mariadb warning"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_mariadb-6bcf7d9595-xg52b_b4836388-ed0e-4cca-81ee-97827274c290/mariadb/0.log
container_id: containerd://c01aa737ade76451d5d7ed72e3605394442f77511e3defd5d1afe680e8662f84
container_image: localhost:5000/mariadb:latest
container_image_id: localhost:5000/mariadb@sha256:249bbb724b9e9c687fde22ea3ef48c6a2bd0f8aa247a3a342a6b4598e5fb627b
container_name: mariadb
pod_name: mariadb-6bcf7d9595-xg52b
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/mariadb-6bcf7d9595
pod_uid: b4836388-ed0e-4cca-81ee-97827274c290
log: "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)"
source_type: kubernetes_logs
stream: stdout
timestamp: "2024-08-19T10:57:36.696731329Z"
outputs:
- extract_from: "parse_unmatched"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "2024-08-19 10:57:36 18532 [Warning] Aborted connection 18532 to db: ''unconnected'' user: ''unauthenticated'' host: ''10.42.0.1'' (This connection closed normally without authentication)")
assert!(is_timestamp(.timestamp))
- name: "Proxy warning"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_proxy-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/proxy/0.log
container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a
container_image: localhost:5000/proxy:latest
container_image_id: localhost:5000/proxy@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68
container_name: proxy
pod_name: proxy-84bb89799b-rflm6
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/proxy-84bb89799b
pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3
log: '2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T10:57:25.026161935Z"
outputs:
- extract_from: "parse_proxy"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "2024/08/19 10:57:25 [info] 11#11: *18701 client closed connection while SSL handshaking, client: 10.42.0.1, server: 0.0.0.0:6443")
assert!(is_timestamp(.timestamp))
- name: "Postfix"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_postfix-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/postfix/0.log
container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a
container_image: localhost:5000/postfix:latest
container_image_id: localhost:5000/postfix@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68
container_name: postfix
pod_name: postfix-5fbfb788c-mjjxh
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/postfix-84bb89799b
pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3
log: 'EB6AA233B6D2: to=<admin@single.k3s.klab.cc>, relay=imap[10.43.9.216]:11024, delay=0.03, delays=0.01/0/0/0.02, dsn=2.1.5, status=sent (250 2.1.5 Success SESSIONID=<cyrus-imapd-1741268864-161809-1-11604045405236919686>)'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T10:57:25.026161935Z"
outputs:
- extract_from: "parse_postfix"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "EB6AA233B6D2: to=<admin@single.k3s.klab.cc>, relay=imap[10.43.9.216]:11024, delay=0.03, delays=0.01/0/0/0.02, dsn=2.1.5, status=sent (250 2.1.5 Success SESSIONID=<cyrus-imapd-1741268864-161809-1-11604045405236919686>)")
assert!(is_timestamp(.timestamp))
- name: "Postfix smtp queued"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_postfix-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/postfix/0.log
container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a
container_image: localhost:5000/postfix:latest
container_image_id: localhost:5000/postfix@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68
container_name: postfix
pod_name: postfix-5fbfb788c-mjjxh
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/postfix-84bb89799b
pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3
log: 'Mar 06 13:20:09 postfix postfix/smtp[68356]: AEAFC233B6D2: to=<test2@single.k3s.klab.cc>, relay=127.0.0.1[127.0.0.1]:13026, delay=7.7, delays=2.4/0.04/0.01/5.2, dsn=2.0.0, status=sent (250 2.0.0 from MTA(smtp:[127.0.0.1]:13025): 250 2.0.0 Ok: queued as 231D4238BAE4)'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T10:57:25.026161935Z"
outputs:
- extract_from: "parse_postfix"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "AEAFC233B6D2: to=<test2@single.k3s.klab.cc>, relay=127.0.0.1[127.0.0.1]:13026, delay=7.7, delays=2.4/0.04/0.01/5.2, dsn=2.0.0, status=sent (250 2.0.0 from MTA(smtp:[127.0.0.1]:13025): 250 2.0.0 Ok: queued as 231D4238BAE4)")
assert!(is_timestamp(.timestamp))
assert_eq!(.process, "postfix/smtp[68356]")
- name: "Postfix saslauthd"
inputs:
- insert_at: "apps"
type: "log"
log_fields:
file: /var/log/pods/kolab_postfix-84bb89799b-rflm6_f0c38b2a-56ab-4417-8bde-d1729ce07ad3/postfix/0.log
container_id: containerd://da6f4f7f22a70f2a21042d642ad40564a41f5731e145de4ee6dffc35de6a543a
container_image: localhost:5000/postfix:latest
container_image_id: localhost:5000/postfix@sha256:4ff5c6c5da8b5e73ecc4f17cba99e00c6dd58e07cce54f04aefd4f0b6063da68
container_name: postfix
pod_name: postfix-5fbfb788c-mjjxh
pod_namespace: kolab
pod_node_name: dws07.kolabsys.com
pod_owner: ReplicaSet/postfix-84bb89799b
pod_uid: f0c38b2a-56ab-4417-8bde-d1729ce07ad3
log: 'saslauthd[471] :auth success: [user=admin] [service=smtp] [realm=single.k3s.klab.cc] [mech=httpform]'
source_type: kubernetes_logs
stream: stderr
timestamp: "2024-08-19T10:57:25.026161935Z"
outputs:
- extract_from: "parse_postfix"
conditions:
- type: "vrl"
source: |
assert_eq!(.log, "saslauthd[471] :auth success: [user=admin] [service=smtp] [realm=single.k3s.klab.cc] [mech=httpform]")
assert!(is_timestamp(.timestamp))
diff --git a/docker/vector/rootfs/config/transforms.yaml b/docker/vector/rootfs/config/transforms.yaml
index a68e38ab..29f94d54 100644
--- a/docker/vector/rootfs/config/transforms.yaml
+++ b/docker/vector/rootfs/config/transforms.yaml
@@ -1,69 +1,69 @@
transforms:
apps:
type: route
inputs:
- input
reroute_unmatched: true
route:
roundcube: contains!(.pod_name, "roundcube")
proxy: contains!(.pod_name, "proxy")
kolab: contains!(.pod_name, "kolab")
postfix: contains!(.pod_name, "postfix")
imap: contains!(.pod_name, "imap")
parse_kolab:
type: remap
inputs:
- apps.kolab
source: |
structured = parse_logfmt(.log) ?? {}
. = merge(., structured)
.timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()
parse_roundcube:
type: remap
inputs:
- apps.roundcube
source: |
structured = parse_apache_log(.log, format: "common") ?? parse_logfmt(.log) ?? {}
. = merge(., structured)
.timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()
parse_imap:
type: remap
inputs:
- apps.imap
source: |
# Drop noisy saslauthd debug messages
if contains(string!(.log), "accept lock") {
abort
}
- structured = parse_regex(.log, pattern: r'^(?<program>.*)\[(?<pid>[0-9]+)\]( )?: *(?<log>.*)$') ?? {}
+ structured = parse_regex(.log, pattern: r'^(?<time>[^ ]+ [0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]) (?<host>[^ ]+) (?<facility>[^ ]+) (?<program>.*)\[(?<pid>[0-9]+)\]( )?: *(?<log>.*)$') ?? {}
. = merge(., structured)
.timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()
parse_proxy:
type: remap
inputs:
- apps.proxy
source: |
structured = parse_regex(.log, pattern: r'^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")') ?? {}
. = merge(., structured)
.timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()
parse_postfix:
type: remap
inputs:
- apps.postfix
source: |
if .container_name == "metricsexporter" {
abort
}
# Drop noisy saslauthd debug messages
if contains(string!(.log), "accept lock") {
abort
}
structured = parse_regex(.log, pattern: r'^(?<time>[^ ]+ [0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]) (?<host>[^ ]+) (?<process>[^:]+): (?<log>((?<key>[^ :]+)[ :])? ?((to|from)=<(?<address>[^>]+)>)?.*)$') ?? {}
. = merge(., structured)
.timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()
parse_unmatched:
type: remap
inputs:
- apps._unmatched
source: |
.timestamp = parse_timestamp(.timestamp, "%Y/%m/%d %H:%M:%S %z") ?? now()