Elasticsearch+Filebeat+Kibana 使用DaemonSet的方式收集Kubernetes 集群中Node节点中的应用服务日志

# cat elasticsearch.yaml 

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  namespace: kube-system
  labels:
    k8s-app: elasticsearch
spec:
  serviceName: elasticsearch
  selector:
    matchLabels:
      k8s-app: elasticsearch
  template:
    metadata:
      labels:
        k8s-app: elasticsearch
    spec:
      initContainers:
      - name: busybox
        imagePullPolicy: IfNotPresent
        image: busybox:latest
        securityContext:
          privileged: true
        command: ["sh", "-c", "mkdir -p /usr/share/elasticsearch/data/logs;chown -R 1000:1000 /usr/share/elasticsearch/data"]
        volumeMounts:
        - name: elasticsearch-data
          mountPath: "/usr/share/elasticsearch/data"

      containers:
      - image: elasticsearch:7.10.1
        name: elasticsearch
        resources:
          limits:
            cpu: 1
            memory: 2Gi
          requests:
            cpu: 0.5 
            memory: 500Mi
        env:
          - name: "discovery.type"
            value: "single-node"
          - name: ES_JAVA_OPTS
            value: "-Xms512m -Xmx2g" 
        ports:
        - containerPort: 9200
          name: db
          protocol: TCP
        volumeMounts:
        - name: elasticsearch-data
          mountPath: /usr/share/elasticsearch/data
  volumeClaimTemplates:
  - metadata:
      name: elasticsearch-data
    spec:
      storageClassName: "disk-sc"
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1000Gi

---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  namespace: kube-system
spec:
  clusterIP: None
  ports:
  - port: 9200
    protocol: TCP
    targetPort: db
  selector:
    k8s-app: elasticsearch

# cat kibana.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: kube-system
  labels:
    k8s-app: kibana
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: kibana
  template:
    metadata:
      labels:
        k8s-app: kibana
    spec:
      containers:
      - name: kibana
        image: kibana:7.10.1
        resources:
          limits:
            cpu: 1
            memory: 500Mi
          requests:
            cpu: 0.5 
            memory: 200Mi
        env:
          - name: ELASTICSEARCH_HOSTS
            value: http://elasticsearch-0.elasticsearch.kube-system:9200
          - name: I18N_LOCALE
            value: zh-CN
        ports:
        - containerPort: 5601
          name: ui
          protocol: TCP

#---
#apiVersion: v1
#kind: Service
#metadata:
#  name: kibana
#  namespace: kube-system
#spec:
#  type: NodePort           # 使用nodeport的service类型
#  ports:
#  - port: 5601
#    protocol: TCP
#    targetPort: ui
#    nodePort: 30601
#  selector:
#    k8s-app: kibana
---
apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: kube-system
  annotations:
    service.kubernetes.io/qcloud-loadbalancer-internal-subnetid: subnet-aabbccdd
spec:
  type: LoadBalancer         # 使用云厂商的clb负载均衡
  ports:
  - name: kibana-pvc
    protocol: TCP
    port: 5601
    targetPort: 5601
  selector:
    k8s-app: kibana

# cat filebeat-kubernetes.yaml 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    filebeat.config:
      inputs:
        # Mounted `filebeat-inputs` configmap:
        path: ${path.config}/inputs.d/*.yml
        # Reload inputs configs as they change:
        reload.enabled: false
      modules:
        path: ${path.config}/modules.d/*.yml
        # Reload module configs as they change:
        reload.enabled: false
    filebeat.autodiscover:       # 使用filebeat自动发现的方式
      providers:
        - type: kubernetes
          templates:
            - condition:
                equals:
                  kubernetes.namespace: prod      # 收集prod命名空间的日志
              config:
                - type: log      # 日志类型为log而非docker或者container，因为我们输出的日志非json格式。
                  containers.ids:
                    - "${data.kubernetes.container.id}"
                  paths:
                    # 收集日志的路径，如果设置为"/var/lib/kubelet/pods/**/*info.log"，则会收集多余的日志，使用云k8s，所以非/var/lib/docker/containers/目录
                    - "/var/lib/kubelet/pods/${data.kubernetes.pod.uid}/volumes/kubernetes.io~empty-dir/data-vol/log/java/*/*info.log" 
                  encoding: utf-8
                  scan_frequency: 1s    # 扫描新文件的时间间隔，默认为10秒
                  tail_files: true      # 设置为true时filebeat从每个文件的末尾取新文件，而不是开始。首次启用filebeat时忽略旧的日志，下次起动时关闭此选项。
                  fields_under_root: true      # 设置为true后，fields存储在输出文档的顶级位置
                  fields:
                    type: "prod-info"
        - type: kubernetes
          templates:
            - condition:
                equals:
                  kubernetes.namespace: prod
              config:
                - type: log
                  containers.ids:
                    - "${data.kubernetes.container.id}"
                  paths:
                    - "/var/lib/kubelet/pods/${data.kubernetes.pod.uid}/volumes/kubernetes.io~empty-dir/data-vol/log/java/*/*error.log"
                  encoding: utf-8
                  scan_frequency: 1s
                  tail_files: true
                  fields_under_root: true
                  fields:
                    type: "prod-error"
                  multiline.type: pattern
                  multiline.pattern: ‘^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:‘
                  multiline.negate: false
                  multiline.match: after

        - type: kubernetes
          templates:
            - condition:
                equals:
                  kubernetes.namespace: test
              config:
                - type: log
                  containers.ids:
                    - "${data.kubernetes.container.id}"
                  paths:
                    - "/var/lib/kubelet/pods/${data.kubernetes.pod.uid}/volumes/kubernetes.io~empty-dir/data-vol/log/java/*/*info.log"
                  encoding: utf-8
                  scan_frequency: 1s
                  tail_files: true
                  fields_under_root: true
                  fields:
                    type: "test-info"
        - type: kubernetes
          templates:
            - condition:
                equals:
                  kubernetes.namespace: test
              config:
                - type: log
                  containers.ids:
                    - "${data.kubernetes.container.id}"
                  paths:
                    - "/var/lib/kubelet/pods/${data.kubernetes.pod.uid}/volumes/kubernetes.io~empty-dir/data-vol/log/java/*/*error.log"
                  encoding: utf-8
                  scan_frequency: 1s
                  tail_files: true
                  fields_under_root: true
                  fields:
                    type: "test-error"
                  multiline.type: pattern
                  multiline.pattern: ‘^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:‘      # 标准的java多行匹配
                  multiline.negate: false
                  multiline.match: after


    setup.ilm.enabled: false
    output.elasticsearch:
      hosts: [‘${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}‘]
      indices:
        - index: "k8s-test-info-%{+yyyy.MM.dd}"
          when.contains:
            type: "test-info"
        - index: "k8s-test-error-%{+yyyy.MM.dd}"
          when.contains:
            type: "test-error"

        - index: "k8s-prod-info-%{+yyyy.MM.dd}"
          when.contains:
            type: "prod-info"
        - index: "k8s-prod-error-%{+yyyy.MM.dd}"
          when.contains:
            type: "prod-error"

   # output.redis:      # 将日志输出到redis中，再用logstash收日志输出到es中
   #   hosts: ["192.168.5.99:6379"]
   #   db: 0
   #   password: "password" 
   #   key: "default_list"
   #   keys:
   #     - key: "k8s-prod-error-%{+yyyy.MM.dd}"
   #       when.contains:
   #         type: "prod-error"
   #     - key: "k8s-prod-info-%{+yyyy.MM.dd}"
   #       when.contains:
   #         type: "prod-info"

---  # 此configmap没有使用，可以将上面的配置文件粘贴到下面，也可删除
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-inputs
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  kubernetes.yml: |-
    #- type: log
    #  paths:
       # - "/var/lib/kubelet/pods/**/*info.log"
      #processors:
      #  - add_kubernetes_metadata:
       #     default_indexers.enabled: false
       #     default_matchers.enabled: false
       #     in_cluster: true
       #     indexers:
       #       - ip_port:
       #     matchers:
       #       - field_format:
       #           format: ‘%{[destination.ip]}:%{[destination.port]}‘
       #     matchers:
       #       - logs_path:
       #           logs_path: ‘/var/log/pods‘
       #           resource_type: ‘pod‘

---
apiVersion: apps/v1 
kind: DaemonSet
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      containers:
      - name: filebeat
        image: elastic/filebeat:7.10.1
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        env:
        - name: ELASTICSEARCH_HOST
          value: elasticsearch-0.elasticsearch.kube-system 
        - name: ELASTICSEARCH_PORT
          value: "9200"
        securityContext:
          runAsUser: 0
          # If using Red Hat OpenShift uncomment this:
          #privileged: true
        resources:
          limits:
            memory: 6144Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
        - name: config
          mountPath: /etc/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: inputs
          mountPath: /usr/share/filebeat/inputs.d
          readOnly: true
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: varlibkubeletpods
          mountPath: /var/lib/kubelet/pods
          readOnly: true
      volumes:
      - name: config
        configMap:
          defaultMode: 0600
          name: filebeat-config
      - name: varlibkubeletpods
        hostPath:
          path: /var/lib/kubelet/pods
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: inputs
        configMap:
          defaultMode: 0600
          name: filebeat-inputs
      # data folder 存储所有文件的读取状态注册表，因此不会在filebeat pod重启后再次发送所有的文件。
      - name: data
        hostPath:
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    k8s-app: filebeat
---

2021-02-16T12:23:33.920Z	ERROR	[reader_docker_json]	readjson/docker_json.go:204	Parse line error: invalid CRI log format
2021-02-16T12:23:33.920Z	INFO	log/harvester.go:335	Skipping unparsable line in file: /var/lib/kubelet/pods/694538b6-4629-4903-804e-8e9fc36ced4a/volumes/kubernetes.io~empty-dir/data-vol/log/java/daemon/daemon-info.log

  filebeat.yml: |-
    filebeat.inputs:
    - type: container      # 日志类型必须为json格式日志才可以正常收集，k8s转到前台的日志可以正常收集。
      paths:
        - "/var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/data-vol/log/java/*/*.log"