Page MenuHomePhorge
Files F117877655

fluentd-configmap.yaml
No OneTemporary
Actions

Authored By
Unknown
Size
11 KB
Referenced Files
None
Subscribers
None

fluentd-configmap.yaml
View Options

{{- if .Values.fluentbit.enabled -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-logging
{{- if not .Values.openshift }}
namespace: kube-system
{{- end }}
labels:
app: fluentd-logging
data:
fluent-bit.yaml: |
service:
http_server: "on"
Health_Check: "on"
log_level: info
parsers_file: /etc/fluent-bit/parsers.conf
pipeline:
inputs:
{{- if .Values.openshift }}
- name: forward
listen: 0.0.0.0
port: 24224
buffer_chunk_size: "1M"
buffer_max_size: "6M"
Mem_Buf_Limit: "100MB"
- name: http
listen: 0.0.0.0
port: 9880
buffer_chunk_size: "1M"
buffer_max_size: "30M"
Mem_Buf_Limit: "100MB"
- name: syslog
listen: 0.0.0.0
port: 5140
mode: tcp
parser: syslog-rfc3164
Mem_Buf_Limit: "100MB"
{{- else }}
- name: tail
path: /var/log/containers/*.log
path_key: tailed_path
skip_empty_lines: "On"
exclude_path: "/var/log/containers/fluent*,/var/log/containers/loki*,/var/log/containers/coredns*"
tag: "kube.<namespace_name>.<pod_name>.<container_name>"
tag_regex: (?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)
multiline.parser: docker, cri
{{- end }}
filters:
{{- if .Values.openshift }}
- name: nest
match: "*"
operation: lift
nested_under: kubernetes
- name: modify
match: '*'
Rename:
- message log
Remove:
- docker
- pipeline_metadata
- openshift
- viaq_msg_id
- container_image
- container_image_id
- pod_id
- pod_ip
- labels
- master_url
- namespace_id
- namespace_labels
- flat_labels
- level
- log_type
- hostname
- host
{{- else }}
- name: parser
match: "kube.*"
key_name: "tailed_path"
reserve_data: true
parser: kube-path
{{- end }}
- name: parser
match: "*.roundcube*"
key_name: "log"
# The log provides a new log key, so we drop this one
preserve_key: true
reserve_data: true
parser: apache
- name: parser
match: "*.roundcube*"
key_name: "log"
# The log provides a new log key, so we drop this one
preserve_key: false
reserve_data: true
parser: roundcube-logfmt
- name: parser
match: "*.postfix*"
key_name: "log"
preserve_key: true
reserve_data: true
parser: maillog
- name: parser
match: "*.imap*"
key_name: "log"
preserve_key: true
reserve_data: true
parser: imap
- name: parser
match: "*.proxy*"
key_name: "log"
preserve_key: true
reserve_data: true
parser: nginx
#user and component may be empty or absent, which triggers a warning in the output plugin,
#so we fill it with dummy values if absent
- name: modify
match: "*"
Condition:
- Key_Does_Not_Exist user
Set: user "null"
- name: modify
match: "*"
Condition:
- Key_Does_Not_Exist component
Set: component "null"
outputs:
# - name: stdout
# match: '*'
- name: loki
match: '*'
{{- if .Values.openshift }}
# Running in the same namespace
host: 'loki'
{{- else }}
# Running in the another namespace, so refer to default kolab namespace
host: 'loki.kolab'
{{- end }}
port: 3100
labels: $pod_name, $namespace_name, $user, $component
parsers.conf: |
[PARSER]
Name imap
Format regex
Regex ^(?<program>.*)\[(?<pid>[0-9]+)\]( )?: *(?<message>.*)$
[PARSER]
Name apache
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache2
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache_error
Format regex
Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
[PARSER]
Name nginx
Format regex
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
# https://rubular.com/r/IhIbCAIs7ImOkc
Name k8s-nginx-ingress
Format regex
Regex ^(?<host>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name json
Format json
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name logfmt
Format logfmt
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
# --
# Since Fluent Bit v1.2, if you are parsing Docker logs and using
# the Kubernetes filter, it's not longer required to decode the
# 'log' key.
#
# Command | Decoder | Field | Optional Action
# =============|==================|=================
#Decode_Field_As json log
[PARSER]
Name docker-daemon
Format regex
Regex time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)"
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
[PARSER]
Name syslog-rfc5424
Format regex
Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
[PARSER]
Name syslog-rfc3164-local
Format regex
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
Time_Key time
Time_Format %b %d %H:%M:%S
Time_Keep On
[PARSER]
Name syslog-rfc3164
Format regex
Regex /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
Time_Key time
Time_Format %b %d %H:%M:%S
Time_Keep On
[PARSER]
Name mongodb
Format regex
Regex ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
Time_Key time
[PARSER]
# https://rubular.com/r/0VZmcYcLWMGAp1
Name envoy
Format regex
Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<code>[^ ]*) (?<response_flags>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<request_id>[^\"]*)" "(?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)"
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
Time_Key start_time
[PARSER]
# https://rubular.com/r/17KGEdDClwiuDG
Name istio-envoy-proxy
Format regex
Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<response_code>[^ ]*) (?<response_flags>[^ ]*) (?<response_code_details>[^ ]*) (?<connection_termination_details>[^ ]*) (?<upstream_transport_failure_reason>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" (?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)" (?<upstream_cluster>[^ ]*) (?<upstream_local_address>[^ ]*) (?<downstream_local_address>[^ ]*) (?<downstream_remote_address>[^ ]*) (?<requested_server_name>[^ ]*) (?<route_name>[^ ]*)
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
Time_Key start_time
[PARSER]
# http://rubular.com/r/tjUt3Awgg4
Name cri
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
[PARSER]
Name kube-custom
Format regex
Regex (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
[PARSER]
Name kube-custom2
Format regex
Regex (?<namespace_name>[^\.]+).(?<pod_name>[^\.]+).(?<container_name>[^\.]+)-(?<docker_id>[a-z0-9]{64})\.log$
[PARSER]
Name kube-path
Format regex
Regex \/(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)\.log$
[PARSER]
Name maillog
Format regex
Regex ^(?<time>[^ ]+ [0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]) (?<host>[^ ]+) (?<process>[^:]+): (?<message>((?<key>[^ :]+)[ :])? ?((to|from)=<(?<address>[^>]+)>)?.*)$
[PARSER]
Name roundcube-logfmt
Format logfmt
Decode_Field_As escaped log
Logfmt_No_Bare_Keys true
{{- end }}

File Metadata

Mime Type
text/plain
Expires
Sun, Apr 5, 9:36 PM (3 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18831268
Default Alt Text
fluentd-configmap.yaml (11 KB)

Event Timeline