add filebeat

fklek/app.yaml

@@ -0,0 +1,122 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: app
+    env: release
+spec:
+  selector:
+    matchLabels:
+      app: app
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
+  # minReadySeconds: 30
+  template:
+    metadata:
+      labels:
+        app: app
+    spec:
+      containers:
+        - name: filebeat                        
+          image: dotbalo/filebeat:5.5.2 
+          resources:
+            requests:
+              memory: "100Mi"
+              cpu: "10m"
+            limits:
+              cpu: "200m"
+              memory: "300Mi"
+          imagePullPolicy: Always
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+            - name: pod_name
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: TZ
+              value: "Asia/Shanghai"
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - name: logpath
+              mountPath: /data/log/app/
+            - name: filebeatconf
+              mountPath: /filebeat.yml 
+              subPath: usr/share/filebeat/filebeat.yml
+        - name: app
+          image: alpine:3.6 
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: logpath
+              mountPath: /home/tomcat/target/
+            - name: tz-config
+              mountPath: /etc/localtime
+            - mountPath: /usr/share/zoneinfo/Asia/Shanghai
+              name: tz-config
+            - mountPath: /etc/timezone
+              name: timezone
+          env:
+            - name: TZ
+              value: "Asia/Shanghai"
+            - name: LANG
+              value: C.UTF-8
+            - name: LC_ALL
+              value: C.UTF-8
+            - name: ENV
+              value: k8srelease
+            - name: XMS
+              value: "2048m"
+            - name: XMX
+              value: "2048m"
+            - name: MEMORY_LIMIT
+              valueFrom:
+                resourceFieldRef:
+                  resource: requests.memory
+                  divisor: 1Mi
+          command:
+            - sh
+            - -c
+            - sleep 360000
+          ports:
+            - containerPort: 8080
+              name: tomcat
+#          livenessProbe:
+#            tcpSocket:
+#              port: tomcat
+#            initialDelaySeconds: 180
+#            timeoutSeconds: 2
+#            failureThreshold: 2
+#          readinessProbe:
+#            tcpSocket:
+#              port: tomcat
+#            initialDelaySeconds: 180
+#            timeoutSeconds: 2
+#            failureThreshold: 2
+      imagePullSecrets:
+        - name: myregistrykey
+      volumes:
+        - name: tz-config
+          hostPath:
+            path: /usr/share/zoneinfo/Asia/Shanghai
+        - hostPath:
+            path: /etc/timezone
+            type: ""
+          name: timezone
+        - name: logpath
+          emptyDir: {}
+        - name: filebeatconf
+          configMap:
+            name: filebeatconf
+            items:
+              - key: filebeat.yml
+                path: usr/share/filebeat/filebeat.yml

fklek/es-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch-logging
+  namespace: logging
+  labels:
+    k8s-app: elasticsearch-logging
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "Elasticsearch"
+spec:
+  ports:
+  - port: 9200
+    protocol: TCP
+    targetPort: db
+  clusterIP: None
+  selector:
+    k8s-app: elasticsearch-logging

fklek/es-ss.yaml

@@ -0,0 +1,119 @@
+# RBAC authn and authz
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: elasticsearch-logging
+  namespace: logging
+  labels:
+    k8s-app: elasticsearch-logging
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: elasticsearch-logging
+  labels:
+    k8s-app: elasticsearch-logging
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "services"
+  - "namespaces"
+  - "endpoints"
+  verbs:
+  - "get"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: logging
+  name: elasticsearch-logging
+  labels:
+    k8s-app: elasticsearch-logging
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+- kind: ServiceAccount
+  name: elasticsearch-logging
+  namespace: logging
+  apiGroup: ""
+roleRef:
+  kind: ClusterRole
+  name: elasticsearch-logging
+  apiGroup: ""
+---
+# Elasticsearch deployment itself
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: elasticsearch-logging
+  namespace: logging
+  labels:
+    k8s-app: elasticsearch-logging
+    version: v5.6.4
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  serviceName: elasticsearch-logging
+  replicas: 3
+  selector:
+    matchLabels:
+      k8s-app: elasticsearch-logging
+      version: v5.6.4
+  template:
+    metadata:
+      labels:
+        k8s-app: elasticsearch-logging
+        version: v5.6.4
+        kubernetes.io/cluster-service: "true"
+    spec:
+      serviceAccountName: elasticsearch-logging
+      volumes:
+        - name: elasticsearch-logging
+          emptyDir: {}
+      containers:
+      - image: dotbalo/elasticsearch:v5.6.4 
+        name: elasticsearch-logging
+        resources:
+          # need more cpu upon initialization, therefore burstable class
+          limits:
+            cpu: 1000m
+          requests:
+            cpu: 100m
+        ports:
+        - containerPort: 9200
+          name: db
+          protocol: TCP
+        - containerPort: 9300
+          name: transport
+          protocol: TCP
+        volumeMounts:
+        - name: elasticsearch-logging
+          mountPath: /data
+        env:
+        - name: "NAMESPACE"
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      # Elasticsearch requires vm.max_map_count to be at least 262144.
+      # If your OS already sets up this number to a higher value, feel free
+      # to remove this init container.
+      initContainers:
+      - image: alpine:3.6 
+        command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
+        name: elasticsearch-logging-init
+        securityContext:
+          privileged: true
+#  volumeClaimTemplates:
+#  - metadata:
+#      name: elasticsearch-logging
+#    spec:
+#      accessModes: [ "ReadWriteOnce" ]
+#      storageClassName: "es-storage-class"
+#      resources:
+#        requests:
+#          storage: 10Gi

fklek/filebeat-configmap.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeatconf
+data:
+  filebeat.yml: |-
+    filebeat.prospectors:
+    - input_type: log
+      paths:
+        - /data/log/*/*.log
+      document_type: "logging"
+      tail_files: true
+      fields:
+        pod_name: '${pod_name}'
+        POD_IP: '${POD_IP}'
+        type: 'logging'
+    
+    output.kafka:
+      hosts: ["kafka-1-ip-service.logging:9092","kafka-2-ip-service.logging:9092","kafka-3-ip-service.logging:9092"]
+      topic: "logging"
+      keep_alive: 30s

fklek/kafka-1-deployment.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-1-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: kafka-1
+  template:
+    metadata:
+      labels:
+        component: kafka-1
+    spec:
+      containers:
+      - name: kafka-1 
+        image: dotbalo/kafka_k8s:v1
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "250m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        ports:
+          - containerPort: 9092
+        env:
+          - name: MY_POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: KAFKA_ADVERTISED_PORT
+            value: "9092"
+          - name: KAFKA_ZOOKEEPER_CONNECT
+            value: zk-0.zk-headless:2181,zk-1.zk-headless:2181,zk-2.zk-headless:2181
+          - name: KAFKA_ADVERTISED_PORT
+            value: "9092"
+          - name: KAFKA_ADVERTISED_HOST_NAME
+            value: $(MY_POD_IP)
+        tty: true
+        livenessProbe:
+          exec:
+            command:
+              - /opt/check.sh
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        readinessProbe:
+          exec:
+            command:
+              - /opt/check.sh
+          initialDelaySeconds: 30
+          periodSeconds: 5
+        # command:
+        #   - /bin/bash
+      imagePullSecrets:
+        - name: regcred
+      

fklek/kafka-1-ip-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-1-ip-service
+spec:
+  type: ClusterIP 
+  selector:
+    component: kafka-1
+  ports:
+  - name: kafka
+    port: 9092
+    targetPort: 9092
+
+
+
+  

fklek/kafka-2-deployment.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-2-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: kafka-2
+  template:
+    metadata:
+      labels:
+        component: kafka-2
+    spec:
+      containers:
+      - name: kafka-2 
+        image: dotbalo/kafka_k8s:v1
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "250m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        ports:
+          - containerPort: 9092
+
+        env:
+          - name: MY_POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: KAFKA_ADVERTISED_PORT
+            value: "9092"
+          - name: KAFKA_ZOOKEEPER_CONNECT
+            value: zk-0.zk-headless:2181,zk-1.zk-headless:2181,zk-2.zk-headless:2181
+          - name: KAFKA_ADVERTISED_PORT
+            value: "9092"
+          - name: KAFKA_ADVERTISED_HOST_NAME
+            value: $(MY_POD_IP)
+        tty: true
+        livenessProbe:
+          exec:
+            command:
+              - /opt/check.sh
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        readinessProbe:
+          exec:
+            command:
+              - /opt/check.sh
+          initialDelaySeconds: 30
+          periodSeconds: 5
+       
+      imagePullSecrets:
+        - name: regcred
+      

fklek/kafka-2-ip-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-2-ip-service
+spec:
+  type: ClusterIP 
+  selector:
+    component: kafka-2
+  ports:
+  - name: kafka
+    port: 9092
+    targetPort: 9092
+
+
+
+  

fklek/kafka-3-deployment.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-3-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: kafka-3
+  template:
+    metadata:
+      labels:
+        component: kafka-3
+    spec:
+      containers:
+      - name: kafka-3 
+        image: dotbalo/kafka_k8s:v1
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "250m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        ports:
+          - containerPort: 9092
+
+        env:
+          - name: MY_POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: KAFKA_ADVERTISED_PORT
+            value: "9092"
+          - name: KAFKA_ZOOKEEPER_CONNECT
+            value: zk-0.zk-headless:2181,zk-1.zk-headless:2181,zk-2.zk-headless:2181
+          - name: KAFKA_ADVERTISED_PORT
+            value: "9092"
+          - name: KAFKA_ADVERTISED_HOST_NAME
+            value: $(MY_POD_IP)
+        tty: true
+        livenessProbe:
+          exec:
+            command:
+              - /opt/check.sh
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        readinessProbe:
+          exec:
+            command:
+              - /opt/check.sh
+          initialDelaySeconds: 30
+          periodSeconds: 5
+       
+      imagePullSecrets:
+        - name: regcred
+      

fklek/kafka-3-ip-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-3-ip-service
+spec:
+  type: ClusterIP 
+  selector:
+    component: kafka-3
+  ports:
+  - name: kafka
+    port: 9092
+    targetPort: 9092
+
+
+
+  

fklek/logstash-cm.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-configmap
+data:
+  logstash.yml: |
+    http.host: "0.0.0.0"
+    path.config: /usr/share/logstash/pipeline
+  logstash.conf: |
+    # all input will come from filebeat, no local logs
+    input {
+      kafka {
+              enable_auto_commit => true
+              auto_commit_interval_ms => "1000"
+              bootstrap_servers => "kafka-1-ip-service:9092,kafka-2-ip-service:9092,kafka-3-ip-service:9092"
+              topics => ["logging"]
+              type => ["logging"]
+          }
+    }
+
+    output {
+       stdout{ codec=>rubydebug}
+       if [type] == "logging"{
+           elasticsearch {
+             hosts => ["elasticsearch-logging-0.elasticsearch-logging:9200","elasticsearch-logging-1.elasticsearch-logging:9200","elasticsearch-logging-2.elasticsearch-logging:9200"]
+             index => "logging-%{+YYYY.MM.dd}"
+          }
+       } else{
+          elasticsearch {
+             hosts => ["elasticsearch-logging-0.elasticsearch-logging:9200","elasticsearch-logging-1.elasticsearch-logging:9200","elasticsearch-logging-2.elasticsearch-logging:9200"]
+             index => "self-sp-%{+YYYY.MM.dd}"
+          }
+       }
+    }

fklek/logstash-service.yaml

@@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: logstash-service
+spec:
+  selector:
+    app: logstash
+  ports:
+  - protocol: TCP
+    port: 5044
+    targetPort: 5044
+  type: ClusterIP

fklek/logstash.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logstash-deployment
+spec:
+  selector:
+    matchLabels:
+      app: logstash
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: logstash
+    spec:
+      containers:
+      - name: logstash
+        image: dotbalo/logstash:5.5-alpine 
+        ports:
+        - containerPort: 5044
+        volumeMounts:
+          - name: config-volume
+            mountPath: /usr/share/logstash/config
+          - name: logstash-pipeline-volume
+            mountPath: /usr/share/logstash/pipeline
+      volumes:
+      - name: config-volume
+        configMap:
+          name: logstash-configmap
+          items:
+            - key: logstash.yml
+              path: logstash.yml
+      - name: logstash-pipeline-volume
+        configMap:
+          name: logstash-configmap
+          items:
+            - key: logstash.conf
+              path: logstash.conf