add filebeat 6.x

fklek/6.x/app.yaml

@@ -0,0 +1,129 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: app
+    env: release
+spec:
+  selector:
+    matchLabels:
+      app: app
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
+  # minReadySeconds: 30
+  template:
+    metadata:
+      labels:
+        app: app
+    spec:
+      containers:
+        - name: filebeat                        
+          image: 10.120.116.178/public-service/filebeat:6.4.3 
+          resources:
+            requests:
+              memory: "100Mi"
+              cpu: "10m"
+            limits:
+              cpu: "200m"
+              memory: "300Mi"
+          imagePullPolicy: Always
+          env:
+            - name: podIp
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+            - name: podName
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: podNamespace
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: podDeployName
+              value: pscm-metadata
+            - name: TZ
+              value: "Asia/Shanghai"
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - name: logpath
+              mountPath: /data/log/app/
+            - name: filebeatconf
+              mountPath: /usr/share/filebeat/filebeat.yml 
+              subPath: usr/share/filebeat/filebeat.yml
+        - name: app
+          image: alpine:3.6 
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: logpath
+              mountPath: /home/tomcat/target/
+            - name: tz-config
+              mountPath: /etc/localtime
+            - mountPath: /usr/share/zoneinfo/Asia/Shanghai
+              name: tz-config
+            - mountPath: /etc/timezone
+              name: timezone
+          env:
+            - name: TZ
+              value: "Asia/Shanghai"
+            - name: LANG
+              value: C.UTF-8
+            - name: LC_ALL
+              value: C.UTF-8
+            - name: ENV
+              value: k8srelease
+            - name: XMS
+              value: "2048m"
+            - name: XMX
+              value: "2048m"
+            - name: MEMORY_LIMIT
+              valueFrom:
+                resourceFieldRef:
+                  resource: requests.memory
+                  divisor: 1Mi
+          command:
+            - sh
+            - -c
+            - sleep 360000
+          ports:
+            - containerPort: 8080
+              name: tomcat
+#          livenessProbe:
+#            tcpSocket:
+#              port: tomcat
+#            initialDelaySeconds: 180
+#            timeoutSeconds: 2
+#            failureThreshold: 2
+#          readinessProbe:
+#            tcpSocket:
+#              port: tomcat
+#            initialDelaySeconds: 180
+#            timeoutSeconds: 2
+#            failureThreshold: 2
+      imagePullSecrets:
+        - name: myregistrykey
+      volumes:
+        - name: tz-config
+          hostPath:
+            path: /usr/share/zoneinfo/Asia/Shanghai
+        - hostPath:
+            path: /etc/timezone
+            type: ""
+          name: timezone
+        - name: logpath
+          emptyDir: {}
+        - name: filebeatconf
+          configMap:
+            name: filebeatconf
+            items:
+              - key: filebeat.yml
+                path: usr/share/filebeat/filebeat.yml

fklek/6.x/es-sts.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: es-cluster
+  namespace: logging
+spec:
+  serviceName: elasticsearch
+  replicas: 3
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      volumes:
+      - name: data
+        emptyDir: {}
+      containers:
+      - name: elasticsearch
+        image: 10.120.116.178/public-service/elasticsearch-oss:6.4.3
+        resources:
+            limits:
+              cpu: 1000m
+            requests:
+              cpu: 100m
+        ports:
+        - containerPort: 9200
+          name: rest
+          protocol: TCP
+        - containerPort: 9300
+          name: inter-node
+          protocol: TCP
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+        env:
+          - name: cluster.name
+            value: k8s-logs
+          - name: node.name
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: discovery.zen.ping.unicast.hosts
+            value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
+          - name: discovery.zen.minimum_master_nodes
+            value: "2"
+          - name: ES_JAVA_OPTS
+            value: "-Xms512m -Xmx512m"
+      initContainers:
+      - name: fix-permissions
+        image: 10.120.116.178/public-service/busybox
+        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+      - name: increase-vm-max-map
+        image: 10.120.116.178/public-service/busybox
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
+      - name: increase-fd-ulimit
+        image: 10.120.116.178/public-service/busybox
+        command: ["sh", "-c", "ulimit -n 65536"]
+        securityContext:
+          privileged: true
+#  volumeClaimTemplates:
+#  #  - metadata:
+#  #      name: data
+#  #      labels:
+#  #        app: elasticsearch
+#  #    spec:
+#  #      accessModes: [ "ReadWriteOnce" ]
+#  #      storageClassName: do-block-storage
+#  #      resources:
+#  #        requests:
+#  #          storage: 100Gi

fklek/6.x/es-svc.yaml

@@ -0,0 +1,16 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: elasticsearch
+  namespace: logging
+  labels:
+    app: elasticsearch
+spec:
+  selector:
+    app: elasticsearch
+  clusterIP: None
+  ports:
+    - port: 9200
+      name: rest
+    - port: 9300
+      name: inter-node

fklek/6.x/filebeat-configmap.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeatconf
+data:
+  filebeat.yml: |-
+    filebeat.prospectors:
+    - input_type: log
+      paths:
+        - /data/log/*/*.log
+      tail_files: true
+      fields:
+        pod_name: '${podName}'
+        pod_ip: '${podIp}'
+        pod_deploy_name: '${podDeployName}'
+        pod_namespace: '${podNamespace}'
+      tags: [pscm-release] 
+    output.kafka:
+      hosts: ["kafka-1-ip-service.logging:9092","kafka-2-ip-service.logging:9092","kafka-3-ip-service.logging:9092"]
+      topic: "pscm-release"
+      codec.json:
+        pretty: false
+      keep_alive: 30s
+

fklek/6.x/kibana.yaml

@@ -0,0 +1,43 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  namespace: logging
+  labels:
+    app: kibana
+spec:
+  ports:
+  - port: 5601
+  selector:
+    app: kibana
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  namespace: logging
+  labels:
+    app: kibana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+      - name: kibana
+        image: 10.120.116.178/public-service/kibana:6.2.4 
+        resources:
+          limits:
+            cpu: 1000m
+          requests:
+            cpu: 100m
+        env:
+          - name: ELASTICSEARCH_URL
+            value: http://elasticsearch:9200
+        ports:
+        - containerPort: 5601

fklek/6.x/logstash-cm.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-configmap
+data:
+  logstash.yml: |
+    http.host: "0.0.0.0"
+    path.config: /usr/share/logstash/pipeline
+  logstash.conf: |
+    # all input will come from filebeat, no local logs
+    input {
+      kafka {
+              enable_auto_commit => true
+              auto_commit_interval_ms => "1000"
+              bootstrap_servers => "kafka-1-ip-service:9092,kafka-2-ip-service:9092,kafka-3-ip-service:9092"
+              topics => ["pscm-release"]
+              codec => json
+              #type => ["pscm-release"]
+          }
+    }
+#    filter {
+#      if [fields][pod_deploy_name] != "" {
+#          mutate {
+#             #add_field => ["[fields][pod_namespace]","%{[fields][pod_deploy_name]}"]
+#             add_field => { "namespace" => "%{[fields][pod_namespace]}" }
+#             add_field => { "deployment" => "%{[fields][pod_deploy_name]}" }
+#        }
+#      }
+#    }
+
+    output {
+       stdout{ codec=>rubydebug}
+       if [fields][pod_namespace] =~ "logging" {
+           elasticsearch {
+             hosts => ["elasticsearch:9200"]
+             index => "%{[fields][pod_namespace]}-s-%{+YYYY.MM.dd}"
+          }
+       } else {
+          elasticsearch {
+             hosts => ["elasticsearch:9200"]
+             index => "no-index-%{+YYYY.MM.dd}"
+          }
+       }
+    }

fklek/6.x/logstash-service.yaml

@@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: logstash-service
+spec:
+  selector:
+    app: logstash
+  ports:
+  - protocol: TCP
+    port: 5044
+    targetPort: 5044
+  type: ClusterIP

fklek/6.x/logstash.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: logstash-deployment
+spec:
+  selector:
+    matchLabels:
+      app: logstash
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: logstash
+    spec:
+      containers:
+      - name: logstash
+        image: 10.120.116.178/public-service/logstash:6.4.3 
+        ports:
+        - containerPort: 5044
+        volumeMounts:
+          - name: config-volume
+            mountPath: /usr/share/logstash/config
+          - name: logstash-pipeline-volume
+            mountPath: /usr/share/logstash/pipeline
+      volumes:
+      - name: config-volume
+        configMap:
+          name: logstash-configmap
+          items:
+            - key: logstash.yml
+              path: logstash.yml
+      - name: logstash-pipeline-volume
+        configMap:
+          name: logstash-configmap
+          items:
+            - key: logstash.conf
+              path: logstash.conf