add alter by email

prometheus-operator/alertmanager.yaml

@@ -0,0 +1,91 @@
+# global块配置下的配置选项在本配置文件内的所有配置项下可见
+global:
+  # 在Alertmanager内管理的每一条告警均有两种状态: "resolved"或者"firing". 在altermanager首次发送告警通知后, 该告警会一直处于firing状态,设置resolve_timeout可以指定处于firing状态的告警间隔多长时间会被设置为resolved状态, 在设置为resolved状态的告警后,altermanager不会再发送firing的告警通知.
+  resolve_timeout: 1h
+
+  # 邮件告警配置
+  smtp_smarthost: 'smtp.exmail.qq.com:25'
+  smtp_from: 'dukuan@xxx.com'
+  smtp_auth_username: 'dukuan@xxx.com'
+  smtp_auth_password: 'DKhxlc2018,./!@#'
+  # HipChat告警配置
+  # hipchat_auth_token: '123456789'
+  # hipchat_auth_url: 'https://hipchat.foobar.org/'
+
+  # 告警通知模板
+templates:
+- '/etc/altermanager/config/*.tmpl'
+
+# route: 根路由,该模块用于该根路由下的节点及子路由routes的定义. 子树节点如果不对相关配置进行配置，则默认会从父路由树继承该配置选项。每一条告警都要进入route，即要求配置选项group_by的值能够匹配到每一条告警的至少一个labelkey(即通过POST请求向altermanager服务接口所发送告警的labels项所携带的<labelname>)，告警进入到route后，将会根据子路由routes节点中的配置项match_re或者match来确定能进入该子路由节点的告警(由在match_re或者match下配置的labelkey: labelvalue是否为告警labels的子集决定，是的话则会进入该子路由节点，否则不能接收进入该子路由节点).
+route:
+  # 例如所有labelkey:labelvalue含cluster=A及altertname=LatencyHigh labelkey的告警都会被归入单一组中
+  group_by: ['job', 'altername', 'cluster', 'service','severity']
+  # 若一组新的告警产生，则会等group_wait后再发送通知，该功能主要用于当告警在很短时间内接连产生时，在group_wait内合并为单一的告警后再发送
+  group_wait: 30s
+  # 再次告警时间间隔
+  group_interval: 5m
+  # 如果一条告警通知已成功发送，且在间隔repeat_interval后，该告警仍然未被设置为resolved，则会再次发送该告警通知
+  repeat_interval: 12h
+  # 默认告警通知接收者，凡未被匹配进入各子路由节点的告警均被发送到此接收者
+  receiver: 'team-ops-mails'
+  # 上述route的配置会被传递给子路由节点，子路由节点进行重新配置才会被覆盖
+
+  # 子路由树
+  routes:
+  # 该配置选项使用正则表达式来匹配告警的labels，以确定能否进入该子路由树
+  # match_re和match均用于匹配labelkey为service,labelvalue分别为指定值的告警，被匹配到的告警会将通知发送到对应的receiver
+  - match_re:
+      service: ^(foo1|foo2|baz)$
+    receiver: 'team-ops-mails'
+    # 在带有service标签的告警同时有severity标签时，他可以有自己的子路由，同时具有severity != critical的告警则被发送给接收者team-ops-mails,对severity == critical的告警则被发送到对应的接收者即team-ops-pager
+    routes:
+    - match:
+        severity: critical
+      receiver: 'team-ops-mails'
+  # 比如关于数据库服务的告警，如果子路由没有匹配到相应的owner标签，则都默认由team-DB-pager接收
+  - match:
+      service: database
+    receiver: 'team-ops-mails'
+  # 我们也可以先根据标签service:database将数据库服务告警过滤出来，然后进一步将所有同时带labelkey为database
+  - match:
+      severity: critical
+    receiver: 'team-ops-mails'
+# 抑制规则，当出现critical告警时 忽略warning
+inhibit_rules:
+- source_match:
+    severity: 'critical'
+  target_match:
+    severity: 'warning'
+  # Apply inhibition if the alertname is the same.
+  #   equal: ['alertname', 'cluster', 'service']
+  #
+# 收件人配置
+receivers:
+- name: 'team-ops-mails'
+  email_configs:
+  - to: 'dukuan@xxx.com'
+
+#- name: 'team-X-pager'
+#  email_configs:
+#  - to: 'team-X+alerts-critical@example.org'
+#  pagerduty_configs:
+#  - service_key: <team-X-key>
+#
+#- name: 'team-Y-mails'
+#  email_configs:
+#  - to: 'team-Y+alerts@example.org'
+#
+#- name: 'team-Y-pager'
+#  pagerduty_configs:
+#  - service_key: <team-Y-key>
+#
+#- name: 'team-DB-pager'
+#  pagerduty_configs:
+#  - service_key: <team-DB-key>
+#  
+#- name: 'team-X-hipchat'
+#  hipchat_configs:
+#  - auth_token: <auth_token>
+#    room_id: 85
+#    message_format: html
+#    notify: true 

prometheus-operator/deploy

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+if [ -z "${KUBECONFIG}" ]; then
+    export KUBECONFIG=~/.kube/config
+fi
+
+# CAUTION - setting NAMESPACE will deploy most components to the given namespace
+# however some are hardcoded to 'monitoring'. Only use if you have reviewed all manifests.
+
+if [ -z "${NAMESPACE}" ]; then
+    NAMESPACE=monitoring
+fi
+
+kubectl create namespace "$NAMESPACE"
+
+kubectl create secret generic  alertmanager-main --from-file=alertmanager.yaml --from-file=mail-template.tmpl -n monitoring
+
+kctl() {
+    kubectl --namespace "$NAMESPACE" "$@"
+}
+
+kctl apply -f manifests/prometheus-operator
+
+# Wait for CRDs to be ready.
+printf "Waiting for Operator to register custom resource definitions..."
+until kctl get customresourcedefinitions servicemonitors.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
+until kctl get customresourcedefinitions prometheuses.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
+until kctl get customresourcedefinitions alertmanagers.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
+until kctl get servicemonitors.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
+until kctl get prometheuses.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
+until kctl get alertmanagers.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
+echo "done!"
+
+kctl apply -f manifests/node-exporter
+kctl apply -f manifests/kube-state-metrics
+kctl apply -f manifests/grafana/grafana-credentials.yaml
+kctl apply -f manifests/grafana
+find manifests/prometheus -type f ! -name prometheus-k8s-roles.yaml ! -name prometheus-k8s-role-bindings.yaml -exec kubectl --namespace "$NAMESPACE" apply -f {} \;
+kubectl apply -f manifests/prometheus/prometheus-k8s-roles.yaml
+kubectl apply -f manifests/prometheus/prometheus-k8s-role-bindings.yaml
+kctl apply -f manifests/alertmanager/
+

prometheus-operator/mail-template.tmpl

@@ -0,0 +1,365 @@
+<!--
+Style and HTML derived from https://github.com/mailgun/transactional-email-templates
+The MIT License (MIT)
+Copyright (c) 2014 Mailgun
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta name="viewport" content="width=device-width" />
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<title>{{ template "__subject" . }}</title>
+<style>
+/* -------------------------------------
+    GLOBAL
+    A very basic CSS reset
+------------------------------------- */
+* {
+  margin: 0;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  box-sizing: border-box;
+  font-size: 14px;
+}
+img {
+  max-width: 100%;
+}
+body {
+  -webkit-font-smoothing: antialiased;
+  -webkit-text-size-adjust: none;
+  width: 100% !important;
+  height: 100%;
+  line-height: 1.6em;
+  /* 1.6em * 14px = 22.4px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
+  /*line-height: 22px;*/
+}
+/* Let's make sure all tables have defaults */
+table td {
+  vertical-align: top;
+}
+/* -------------------------------------
+    BODY & CONTAINER
+------------------------------------- */
+body {
+  background-color: #f6f6f6;
+}
+.body-wrap {
+  background-color: #f6f6f6;
+  width: 100%;
+}
+.container {
+  display: block !important;
+  max-width: 600px !important;
+  margin: 0 auto !important;
+  /* makes it centered */
+  clear: both !important;
+}
+.content {
+  max-width: 600px;
+  margin: 0 auto;
+  display: block;
+  padding: 20px;
+}
+/* -------------------------------------
+    HEADER, FOOTER, MAIN
+------------------------------------- */
+.main {
+  background-color: #fff;
+  border: 1px solid #e9e9e9;
+  border-radius: 3px;
+}
+.content-wrap {
+  padding: 30px;
+}
+.content-block {
+  padding: 0 0 20px;
+}
+.header {
+  width: 100%;
+  margin-bottom: 20px;
+}
+.footer {
+  width: 100%;
+  clear: both;
+  color: #999;
+  padding: 20px;
+}
+.footer p, .footer a, .footer td {
+  color: #999;
+  font-size: 12px;
+}
+/* -------------------------------------
+    TYPOGRAPHY
+------------------------------------- */
+h1, h2, h3 {
+  font-family: "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif;
+  color: #000;
+  margin: 40px 0 0;
+  line-height: 1.2em;
+  font-weight: 400;
+}
+h1 {
+  font-size: 32px;
+  font-weight: 500;
+  /* 1.2em * 32px = 38.4px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
+  /*line-height: 38px;*/
+}
+h2 {
+  font-size: 24px;
+  /* 1.2em * 24px = 28.8px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
+  /*line-height: 29px;*/
+}
+h3 {
+  font-size: 18px;
+  /* 1.2em * 18px = 21.6px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
+  /*line-height: 22px;*/
+}
+h4 {
+  font-size: 14px;
+  font-weight: 600;
+}
+p, ul, ol {
+  margin-bottom: 10px;
+  font-weight: normal;
+}
+p li, ul li, ol li {
+  margin-left: 5px;
+  list-style-position: inside;
+}
+/* -------------------------------------
+    LINKS & BUTTONS
+------------------------------------- */
+a {
+  color: #348eda;
+  text-decoration: underline;
+}
+.btn-primary {
+  text-decoration: none;
+  color: #FFF;
+  background-color: #348eda;
+  border: solid #348eda;
+  border-width: 10px 20px;
+  line-height: 2em;
+  /* 2em * 14px = 28px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
+  /*line-height: 28px;*/
+  font-weight: bold;
+  text-align: center;
+  cursor: pointer;
+  display: inline-block;
+  border-radius: 5px;
+  text-transform: capitalize;
+}
+/* -------------------------------------
+    OTHER STYLES THAT MIGHT BE USEFUL
+------------------------------------- */
+.last {
+  margin-bottom: 0;
+}
+.first {
+  margin-top: 0;
+}
+.aligncenter {
+  text-align: center;
+}
+.alignright {
+  text-align: right;
+}
+.alignleft {
+  text-align: left;
+}
+.clear {
+  clear: both;
+}
+/* -------------------------------------
+    ALERTS
+    Change the class depending on warning email, good email or bad email
+------------------------------------- */
+.alert {
+  font-size: 16px;
+  color: #fff;
+  font-weight: 500;
+  padding: 20px;
+  text-align: center;
+  border-radius: 3px 3px 0 0;
+}
+.alert a {
+  color: #fff;
+  text-decoration: none;
+  font-weight: 500;
+  font-size: 16px;
+}
+.alert.alert-warning {
+  background-color: #E6522C;
+}
+.alert.alert-bad {
+  background-color: #D0021B;
+}
+.alert.alert-good {
+  background-color: #68B90F;
+}
+/* -------------------------------------
+    INVOICE
+    Styles for the billing table
+------------------------------------- */
+.invoice {
+  margin: 40px auto;
+  text-align: left;
+  width: 80%;
+}
+.invoice td {
+  padding: 5px 0;
+}
+.invoice .invoice-items {
+  width: 100%;
+}
+.invoice .invoice-items td {
+  border-top: #eee 1px solid;
+}
+.invoice .invoice-items .total td {
+  border-top: 2px solid #333;
+  border-bottom: 2px solid #333;
+  font-weight: 700;
+}
+/* -------------------------------------
+    RESPONSIVE AND MOBILE FRIENDLY STYLES
+------------------------------------- */
+@media only screen and (max-width: 640px) {
+  body {
+    padding: 0 !important;
+  }
+  h1, h2, h3, h4 {
+    font-weight: 800 !important;
+    margin: 20px 0 5px !important;
+  }
+  h1 {
+    font-size: 22px !important;
+  }
+  h2 {
+    font-size: 18px !important;
+  }
+  h3 {
+    font-size: 16px !important;
+  }
+  .container {
+    padding: 0 !important;
+    width: 100% !important;
+  }
+  .content {
+    padding: 0 !important;
+  }
+  .content-wrap {
+    padding: 10px !important;
+  }
+  .invoice {
+    width: 100% !important;
+  }
+}
+</style>
+</head>
+
+<body itemscope itemtype="http://schema.org/EmailMessage">
+
+<table class="body-wrap">
+  <tr>
+    <td></td>
+    <td class="container" width="600">
+      <div class="content">
+        <table class="main" width="100%" cellpadding="0" cellspacing="0">
+          <tr>
+            {{ if gt (len .Alerts.Firing) 0 }}
+            <td class="alert alert-warning">
+            {{ else }}
+            <td class="alert alert-good">
+            {{ end }}
+              {{ .Alerts | len }} alert{{ if gt (len .Alerts) 1 }}s{{ end }} for {{ range .GroupLabels.SortedPairs }}
+                {{ .Name }}={{ .Value }} 
+              {{ end }}
+            </td>
+          </tr>
+          <tr>
+            <td class="content-wrap">
+              <table width="100%" cellpadding="0" cellspacing="0">
+                <tr>
+                  <td class="content-block">
+                    <a href='{{ template "__alertmanagerURL" . }}' class="btn-primary">View in {{ template "__alertmanager" . }}</a>
+                  </td>
+                </tr>
+                {{ if gt (len .Alerts.Firing) 0 }}
+                <tr>
+                  <td class="content-block">
+                    <strong>[{{ .Alerts.Firing | len }}] Firing</strong>
+                  </td>
+                </tr>
+                {{ end }}
+                {{ range .Alerts.Firing }}
+                <tr>
+                  <td class="content-block">
+                    <strong>Labels</strong><br />
+                    {{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
+                    {{ if gt (len .Annotations) 0 }}<strong>Annotations</strong><br />{{ end }}
+                    {{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
+                    <a href="{{ .GeneratorURL }}">Source</a><br />
+                  </td>
+                </tr>
+                {{ end }}
+
+                {{ if gt (len .Alerts.Resolved) 0 }}
+                  {{ if gt (len .Alerts.Firing) 0 }}
+                <tr>
+                  <td class="content-block">
+                    <br />
+                    <hr />
+                    <br />
+                  </td>
+                </tr>
+                  {{ end }}
+                <tr>
+                  <td class="content-block">
+                    <strong>[{{ .Alerts.Resolved | len }}] Resolved</strong>
+                  </td>
+                </tr>
+                {{ end }}
+                {{ range .Alerts.Resolved }}
+                <tr>
+                  <td class="content-block">
+                    <strong>Labels</strong><br />
+                    {{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
+                    {{ if gt (len .Annotations) 0 }}<strong>Annotations</strong><br />{{ end }}
+                    {{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
+                    <a href="{{ .GeneratorURL }}">Source</a><br />
+                  </td>
+                </tr>
+                {{ end }}
+              </table>
+            </td>
+          </tr>
+        </table>
+
+        <div class="footer">
+          <table width="100%">
+            <tr>
+              <td class="aligncenter content-block"><a href='{{ .ExternalURL }}'>Sent by {{ template "__alertmanager" . }}</a></td>
+            </tr>
+          </table>
+        </div></div>
+    </td>
+    <td></td>
+  </tr>
+</table>
+
+</body>
+</html>