Page MenuHomePhorge
Files F117754134

No OneTemporary
Actions

Authored By
Unknown
Size
10 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/docker-compose.build.yml b/docker-compose.build.yml
index 534814c0..7b1a2e75 100644
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@@ -1,32 +1,37 @@
version: '3'
services:
swoole:
build:
context: ./docker/swoole/
image: apheleia/swoole
almalinux8:
build:
context: ./docker/base/
dockerfile: almalinux8
image: apheleia/almalinux8
almalinux9:
build:
context: ./docker/base/
dockerfile: almalinux9
image: apheleia/almalinux9
tests:
build:
context: ./docker/tests/
container_name: kolab-tests
image: kolab-tests
+ fluentbit:
+ build:
+ context: ./docker/fluentbit/
+ container_name: fluentbit
+ image: fluentbit
utils:
build:
context: ./docker/utils/
container_name: kolab-utils
image: kolab-utils
dns: 172.18.0.11
extra_hosts:
- "${APP_DOMAIN}:172.18.0.7"
networks:
kolab:
ipv4_address: 172.18.0.27
diff --git a/docker/fluentbit/Dockerfile b/docker/fluentbit/Dockerfile
new file mode 100644
index 00000000..4a728ebd
--- /dev/null
+++ b/docker/fluentbit/Dockerfile
@@ -0,0 +1,42 @@
+FROM apheleia/almalinux9
+
+RUN dnf -y install ca-certificates gcc gcc-c++ cmake make bash \
+ wget unzip systemd-devel wget flex bison \
+ postgresql-libs postgresql-devel postgresql-server postgresql \
+ cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \
+ libyaml-devel zlib-devel && \
+ dnf clean all
+
+WORKDIR /tmp/fluent-bit/
+ENV VERSION=2.2.2
+RUN wget https://github.com/fluent/fluent-bit/archive/refs/tags/v${VERSION}.tar.gz && tar xzf v${VERSION}.tar.gz && rm *.tar.gz
+RUN cd /tmp/fluent-bit/fluent-bit-${VERSION}/ && \
+ cmake3 -DCMAKE_INSTALL_PREFIX="/opt/fluent-bit/" \
+ -DCMAKE_INSTALL_SYSCONFDIR="/etc" \
+ -DFLB_RELEASE="On" \
+ -DFLB_TRACE="On" \
+ -DFLB_SQLDB="On" \
+ -DFLB_HTTP_SERVER="On" \
+ -DFLB_OUT_KAFKA="On" \
+ -DFLB_OUT_PGSQL="On" \
+ -DFLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" \
+ -DFLB_JEMALLOC="On" \
+ -DFLB_CHUNK_TRACE="On" \
+ ./ && \
+ make -j 4 && \
+ make install && \
+ cd / && \
+ rm -rf /tmp/fluent-bit && \
+ chgrp -R 0 /opt/fluent-bit && \
+ chmod -R g=u /opt/fluent-bit
+
+COPY fluent-bit.yaml /etc/fluent-bit/fluent-bit.yaml
+COPY parsers.conf /etc/fluent-bit/parsers.conf
+
+RUN chgrp -R 0 /etc/fluent-bit && \
+ chmod -R g=u /etc/fluent-bit
+USER 1001
+WORKDIR /
+VOLUME /data
+
+CMD [ "/opt/fluent-bit/bin/fluent-bit", "--config=/etc/fluent-bit/fluent-bit.yaml"]
diff --git a/docker/fluentbit/fluent-bit.yaml b/docker/fluentbit/fluent-bit.yaml
new file mode 100644
index 00000000..8906c467
--- /dev/null
+++ b/docker/fluentbit/fluent-bit.yaml
@@ -0,0 +1,42 @@
+service:
+ http_server: "on"
+ Health_Check: "on"
+ log_level: info
+ parsers_file: /etc/fluent-bit/parsers.conf
+
+pipeline:
+ inputs:
+ - name: tail
+ path: /var/log/containers/*.log
+ path_key: tailed_path
+ exclude_path: "/var/log/containers/fluent*,/var/log/containers/loki*"
+ skip_empty_lines: "On"
+ tag_regex: (?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)\.log$
+ tag: "kube.<namespace_name>.<pod_name>.<container_name>"
+ multiline.parser: docker, cri
+
+ filters:
+ - name: parser
+ match: "kube.*"
+ key_name: "tailed_path"
+ reserve_data: true
+ parser: kube-path
+ - name: parser
+ match: "kube.kolab.roundcube*"
+ key_name: "log"
+ reserve_data: true
+ parser: apache
+ parser: logfmt
+ - name: parser
+ match: "kube.kolab.proxy*"
+ key_name: "log"
+ reserve_data: true
+ parser: nginx
+
+ outputs:
+ - name: stdout
+ match: '*'
+ - name: loki
+ host: 'loki.kolab'
+ port: 3100
+ labels: $pod_name, $namespace_name
diff --git a/docker/fluentbit/parsers.conf b/docker/fluentbit/parsers.conf
new file mode 100644
index 00000000..68c255b1
--- /dev/null
+++ b/docker/fluentbit/parsers.conf
@@ -0,0 +1,140 @@
+[PARSER]
+ Name apache
+ Format regex
+ Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+ Time_Key time
+ Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+ Name apache2
+ Format regex
+ Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$
+ Time_Key time
+ Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+ Name apache_error
+ Format regex
+ Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+[PARSER]
+ Name nginx
+ Format regex
+ Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
+ Time_Key time
+ Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+ # https://rubular.com/r/IhIbCAIs7ImOkc
+ Name k8s-nginx-ingress
+ Format regex
+ Regex ^(?<host>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$
+ Time_Key time
+ Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+ Name json
+ Format json
+ Time_Key time
+ Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+ Name logfmt
+ Format logfmt
+
+[PARSER]
+ Name docker
+ Format json
+ Time_Key time
+ Time_Format %Y-%m-%dT%H:%M:%S.%L
+ Time_Keep On
+ # --
+ # Since Fluent Bit v1.2, if you are parsing Docker logs and using
+ # the Kubernetes filter, it's not longer required to decode the
+ # 'log' key.
+ #
+ # Command | Decoder | Field | Optional Action
+ # =============|==================|=================
+ #Decode_Field_As json log
+
+[PARSER]
+ Name docker-daemon
+ Format regex
+ Regex time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)"
+ Time_Key time
+ Time_Format %Y-%m-%dT%H:%M:%S.%L
+ Time_Keep On
+
+[PARSER]
+ Name syslog-rfc5424
+ Format regex
+ Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
+ Time_Key time
+ Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+ Time_Keep On
+
+[PARSER]
+ Name syslog-rfc3164-local
+ Format regex
+ Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
+ Time_Key time
+ Time_Format %b %d %H:%M:%S
+ Time_Keep On
+
+[PARSER]
+ Name syslog-rfc3164
+ Format regex
+ Regex /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+ Time_Key time
+ Time_Format %b %d %H:%M:%S
+ Time_Keep On
+
+[PARSER]
+ Name mongodb
+ Format regex
+ Regex ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$
+ Time_Format %Y-%m-%dT%H:%M:%S.%L
+ Time_Keep On
+ Time_Key time
+
+[PARSER]
+ # https://rubular.com/r/0VZmcYcLWMGAp1
+ Name envoy
+ Format regex
+ Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<code>[^ ]*) (?<response_flags>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<request_id>[^\"]*)" "(?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)"
+ Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+ Time_Keep On
+ Time_Key start_time
+
+[PARSER]
+ # https://rubular.com/r/17KGEdDClwiuDG
+ Name istio-envoy-proxy
+ Format regex
+ Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<response_code>[^ ]*) (?<response_flags>[^ ]*) (?<response_code_details>[^ ]*) (?<connection_termination_details>[^ ]*) (?<upstream_transport_failure_reason>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" (?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)" (?<upstream_cluster>[^ ]*) (?<upstream_local_address>[^ ]*) (?<downstream_local_address>[^ ]*) (?<downstream_remote_address>[^ ]*) (?<requested_server_name>[^ ]*) (?<route_name>[^ ]*)
+ Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+ Time_Keep On
+ Time_Key start_time
+
+[PARSER]
+ # http://rubular.com/r/tjUt3Awgg4
+ Name cri
+ Format regex
+ Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+ Time_Key time
+ Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+ Time_Keep On
+
+[PARSER]
+ Name kube-custom
+ Format regex
+ Regex (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
+
+[PARSER]
+ Name kube-custom2
+ Format regex
+ Regex (?<namespace_name>[^\.]+).(?<pod_name>[^\.]+).(?<container_name>[^\.]+)-(?<docker_id>[a-z0-9]{64})\.log$
+
+[PARSER]
+ Name kube-path
+ Format regex
+ Regex \/(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)\.log$

File Metadata

Mime Type
text/x-diff
Expires
Sat, Apr 4, 6:38 AM (1 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18822870
Default Alt Text
(10 KB)

Event Timeline