02-基于Kubernetes集群构建Filebeat+Zookeeper+Logstash+ElasticSearch+Kibana架构

架构图

基于上节课EFK架构进行调整

删除fluentd的daemonset

kubectl -n public-service delete ds fluentd-es-v3.0.1

安装zookeeper和kafka，具体安装过程见https://www.ljh.cool/37676.html

kubectl get svc -n public-service

安装filebeat和 logstash

项目：https://github.com/dotbalo/k8s/tree/master/efk-7.10.2/filebeat

mkdir ~/filebeat && cd ~/filebeat

vim filebeat-cm.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeatconf
  namespace: public-service
data:
  filebeat.yml: |-
    filebeat.inputs:
    - input_type: log
      # app日志挂载目录，自己去设定，后面filebeat边车容器收集的日志需要满足共享到此目录
      paths:
        - /data/log/*/*.log
      tail_files: true
      # 以下四个字段将会在es上产生索引
      fields:
        pod_name: '${podName}'
        pod_ip: '${podIp}'
        pod_deploy_name: '${podDeployName}'
        pod_namespace: '${podNamespace}'
    output.kafka:
      # 因为处在同一个namespace中，所以直接填写kafka
      hosts: ["kafka:9092"]
      # 这个topic名字等会在kafka要用到
      topic: "filebeat-sidecar"
      codec.json:
        pretty: false
      keep_alive: 30s

vim logstash-cm.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-configmap
  namespace: public-service
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    path.config: /usr/share/logstash/pipeline
  logstash.conf: |
    # all input will come from filebeat, no local logs
    input {
      kafka {
              enable_auto_commit => true
              auto_commit_interval_ms => "1000"
              bootstrap_servers => "kafka:9092"
              topics => ["filebeat-sidecar"]
              type => ["filebeat-sidecar"]
              codec => json
          }
    }

    output {
       stdout{ codec=>rubydebug}
       if [type] == "filebeat-sidecar"{
           elasticsearch {
             hosts => ["elasticsearch-logging:9200"]
             index => "filebeat-%{+YYYY.MM.dd}"
          }
       } else{
          elasticsearch {
             hosts => ["elasticsearch-logging:9200"]
             index => "other-input-%{+YYYY.MM.dd}"
          }
       }
    }

logstash-service.yaml

kind: Service
apiVersion: v1
metadata:
  name: logstash-service
  namespace: public-service
spec:
  selector:
    app: logstash
  ports:
  - protocol: TCP
    port: 5044
    targetPort: 5044
  type: ClusterIP

logstash.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: logstash-deployment
  namespace: public-service
spec:
  selector:
    matchLabels:
      app: logstash
  replicas: 1
  template:
    metadata:
      labels:
        app: logstash
    spec:
      containers:
      - name: logstash
        image: registry.cn-beijing.aliyuncs.com/dotbalo/logstash:7.10.1 
        ports:
        - containerPort: 5044
        volumeMounts:
          - name: config-volume
            mountPath: /usr/share/logstash/config
          - name: logstash-pipeline-volume
            mountPath: /usr/share/logstash/pipeline
      volumes:
      - name: config-volume
        configMap:
          name: logstash-configmap
          items:
            - key: logstash.yml
              path: logstash.yml
      - name: logstash-pipeline-volume
        configMap:
          name: logstash-configmap
          items:
            - key: logstash.conf
              path: logstash.conf

app-filebeat.yaml（使用边车容器实现共享）

apiVersion: apps/v1
kind: Deployment
metadata:
  name: app
  namespace: public-service
  labels:
    app: app
    env: release
spec:
  selector:
    matchLabels:
      app: app
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
  # minReadySeconds: 30
  template:
    metadata:
      labels:
        app: app
    spec:
      containers:
        - name: filebeat                        
          image: registry.cn-beijing.aliyuncs.com/dotbalo/filebeat:7.10.2 
          resources:
            requests:
              memory: "100Mi"
              cpu: "10m"
            limits:
              cpu: "200m"
              memory: "300Mi"
          imagePullPolicy: IfNotPresent
          # 注入环境变量，形成kibana中的过滤字段
          env:
            - name: podIp
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            - name: podName
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: podNamespace
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: podDeployName
              value: app
            - name: TZ
              value: "Asia/Shanghai"
          securityContext:
            runAsUser: 0
          volumeMounts:
          # filebeat日志收集目录，与自定义日志的目录共享
            - name: logpath
              mountPath: /data/log/app/
            - name: filebeatconf
              mountPath: /usr/share/filebeat/filebeat.yml 
              subPath: usr/share/filebeat/filebeat.yml
        # 创建一个应用程序
        - name: app
          image: registry.cn-beijing.aliyuncs.com/dotbalo/alpine:3.6 
          imagePullPolicy: IfNotPresent
          # 自定义日志生成目录
          volumeMounts:
            - name: logpath
              mountPath: /data/log/app/
          env:
            - name: TZ
              value: "Asia/Shanghai"
            - name: LANG
              value: C.UTF-8
            - name: LC_ALL
              value: C.UTF-8
          command:
            - sh
            - -c
            - while true; do date >> /data/log/app/date.log; sleep 2;  done 
      volumes:
        - name: logpath
          emptyDir: {}
        - name: filebeatconf
          configMap:
            name: filebeatconf
            items:
              - key: filebeat.yml
                path: usr/share/filebeat/filebeat.yml