实验环境

三节点 Kubernetes 1.17.2

• 192.168.220.110 kmaster
• 192.168.220.111 knode1
• 192.168.220.112 knode2

组件

• Prometheus：收集、查询、分析、存储监控数据，触发告警，部署方式为 deployment
• Grafana：图形化展示监控数据，部署方式为 deployment
• Alertmanager：发送告警通知，部署方式为 deployment
• node-exporter：收集节点数据，部署方式为 daemonset
• blackbox-exporter：黑盒监控非 kubernetes 服务，部署方式为 deployment

我在自己的实验环境部署的所有组件，每个组件的 configmap、deployment、service 都写在一个 yaml 文件中一起启停了，为了避免多次下载镜像我绑定了 node，存储都配置的是宿主机本地存储。Prometheus、Alertmanager 和 blackbox-exporter 的 configmap 最好单独部署，这样修改配置后向 /-/reload 端点 post 一下就可以重新加载配置而不用重启 deployment。

部署两个 service 的目的：-inner 用于组件间通信，-server用于外部访问组件的web页面。

全部组件运行在 monitoring 命名空间内。

apiVersion: v1
kind: Namespace
metadata:
   name: monitoring
   labels:
     name: monitoring

Prometheus

配置告警规则，数据采集参数，Alertmanager 地址。除了 Kubernetes 一些标准的监控对象，我额外通过 blackbox-exporter 监控了非Kubernetes 部署的一个 web 服务和一个 logstash 服务，告警的对象包括节点的基础资源和 blackbox-exporter 的两个监控对象。web 界面的 Targets 里除了 kube-state-metrics 都 up 了，这个后面再研究。

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-server-conf
  labels:
    name: prometheus-server-conf
  namespace: monitoring
data:
  prometheus.rules: |-
    groups:
    - name: web_probe
      rules:
       - alert: ProbeFailing
         expr: probe_success == 0
         # expr: up{job="blackbox"} == 0 or probe_success{job="blackbox"} == 0
         for: 10s
         labels:
          severity: critical
          team: blackbox
         annotations:
          summary: "网站/端口 {{ $labels.instance }} 失联 {{ $value }}。"
          description: "Job {{$labels.job}} 中的 网站/端口 {{ $labels.instance }} 已失联超过1分钟。"
    - name: node_alert
      rules:
      - alert: HighNodeMemory
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 5
        # expr: (node_memory_Buffers_bytes+node_memory_Cached_bytes+node_memory_MemFree_bytes)/node_memory_MemTotal_bytes*100 > 90 
        for: 10s
        labels:
          severity: critical
          team: node
        annotations:
          summary: "Instance {{ $labels.instance }} 节点内存使用率超过95%"
          description: "{{ $labels.instance }} of job {{$labels.job}}内存使用率超过95%,当前使用率[{{ $value }}]."
      - alert: HighNodeCpuLoad
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 节点CPU 5分钟负载大于80%"
          description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfRootDiskSpace
        expr: (node_filesystem_avail_bytes{mountpoint="/"}  * 100) / node_filesystem_size_bytes{mountpoint="/"} < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 节点根目录剩余空间小于10%"
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfHomeDiskSpace
        expr: (node_filesystem_avail_bytes{mountpoint="/home"}  * 100) / node_filesystem_size_bytes{mountpoint="/home"} < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 节点家目录剩余空间小于10%"
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfRootInodes
        expr: node_filesystem_files_free{mountpoint ="/"} / node_filesystem_files{mountpoint ="/"} * 100 < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 节点根目录剩余inode小于10%"
          description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfHomeInodes
        expr: node_filesystem_files_free{mountpoint ="/home"} / node_filesystem_files{mountpoint ="/home"} * 100 < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 节点家目录剩余inode小于10%"
          description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

  prometheus.yml: |-
    global:
      scrape_interval: 5s
      evaluation_interval: 5s
    rule_files:
      - /etc/prometheus/prometheus.rules
    alerting:
      alertmanagers:
      - scheme: http
        static_configs:
        - targets: 
          - alertmanager-inner.monitoring.svc:9093
          # - 192.168.220.112:30002
    scrape_configs:
      - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
        - role: endpoints
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https
      - job_name: 'kubernetes-nodes'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics
      
      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name
      
      - job_name: 'kube-state-metrics'
        static_configs:
          - targets: ['kube-state-metrics.kube-system.svc.cluster.local:9090']

      - job_name: 'kubernetes-cadvisor'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
      
      - job_name: 'kubernetes-service-endpoints'
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
          action: replace
          target_label: __scheme__
          regex: (https?)
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
          action: replace
          target_label: __address__
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: kubernetes_name

      - job_name: 'web_status'
        scrape_interval: 5s
        metrics_path: /probe
        params:
          module: [http_2xx]
        static_configs:
            - targets:
              - https://www.baidu.com/
              - http://192.168.220.112:5000/health
        relabel_configs:
            - source_labels: [__address__]
              target_label: __param_target   
            - source_labels: [__param_target]
              target_label: instance         
            - target_label: __address__
              replacement: blackbox-exporter-inner.monitoring.svc:9115

      - job_name: "port_status"
        scrape_interval: 5s
        metrics_path: /probe
        params:
          module: [tcp_connect]
        static_configs:
            - targets: [ '192.168.220.111:30102' ]
        relabel_configs:
            - source_labels: [__address__]
              target_label: __param_target
            - source_labels: [__param_target]
              target_label: instance
            - target_label: __address__
              replacement: blackbox-exporter-inner.monitoring.svc:9115
---        
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-server
  template:
    metadata:
      labels:
        app: prometheus-server
    spec:
      nodeSelector:
        kubernetes.io/hostname: knode2
      containers:
        - name: prometheus
          image: prom/prometheus:v2.16.0
          imagePullPolicy: IfNotPresent
          args:
            - "--config.file=/etc/prometheus/prometheus.yml"
            - "--storage.tsdb.path=/prometheus/"
          ports:
            - containerPort: 9090
          volumeMounts:
            - name: prometheus-config-volume
              mountPath: /etc/prometheus/
            - name: prometheus-storage-volume
              mountPath: /prometheus/              
      volumes:
        - name: prometheus-config-volume
          configMap:
            defaultMode: 420
            name: prometheus-server-conf
        - name: prometheus-storage-volume
          emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus-service
  namespace: monitoring
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port:   '9090'
spec:
  selector: 
    app: prometheus-server
  type: NodePort  
  ports:
    - port: 8080
      targetPort: 9090 
      nodePort: 30000

Grafana

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      name: grafana
      annotations:
        prometheus.io/scrape: "true"
      labels:
        app: grafana
    spec:
      containers:
        - name: grafana
          image: grafana/grafana
          imagePullPolicy: IfNotPresent
          ports:
          - name: grafana
            containerPort: 3000
          env:
            # The following env variables set up basic auth twith the default admin user and admin password.
            - name: GF_AUTH_BASIC_ENABLED
              value: "true"
            - name: GF_AUTH_ANONYMOUS_ENABLED
              value: "false"
            # - name: GF_AUTH_ANONYMOUS_ORG_ROLE
            #   value: Admin
            # does not really work, because of template variables in exported dashboards:
            # - name: GF_DASHBOARDS_JSON_ENABLED
            #   value: "true"
          readinessProbe:
            httpGet:
              path: /login
              port: 3000
            # initialDelaySeconds: 30
            # timeoutSeconds: 1
          volumeMounts:
            - mountPath: /var/lib/grafana
              name: grafana-storage
      volumes:
        - name: grafana-storage
          emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: grafana-service
  namespace: monitoring
spec:
  selector: 
    app: grafana
  type: NodePort
  selector:
    app: grafana
  ports:
    - port: 3000
      targetPort: 3000
      nodePort: 30001

启动以后在grafana里面配置prometheus源。

dashboard:

• 315 kubernetes cluster
• 7587 blackbox exporter

Node-exporter

apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  labels:
    app: node-exporter
  name: node-exporter-inner
  namespace: monitoring
spec:
  clusterIP: None
  ports:
  - name: scrape
    port: 9100
    protocol: TCP
  selector:
    app: node-exporter
  type: ClusterIP
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      labels:
        app: node-exporter
      name: node-exporter
    spec:
      hostNetwork: true
      hostPID: true
      hostIPC: true
      securityContext:
        runAsUser: 0
      containers:
      - image: prom/node-exporter
        imagePullPolicy: IfNotPresent 
        name: node-exporter
        volumeMounts:
          - mountPath: /run/systemd/private
            name: systemd-socket
            readOnly: true
        args:
          - "--collector.systemd"
          - "--collector.systemd.unit-whitelist=(docker|ssh|rsyslog|kubelet).service"
        ports:
          - containerPort: 9100
            hostPort: 9100
            name: scrape
        livenessProbe:
          httpGet:
            path: /metrics
            port: 9100
          initialDelaySeconds: 30
          timeoutSeconds: 10
          periodSeconds: 1
        readinessProbe:
          failureThreshold: 5
          httpGet:
            path: /metrics
            port: 9100
          initialDelaySeconds: 10
          timeoutSeconds: 10
          periodSeconds: 2
      volumes:
        - hostPath:  
            path: /run/systemd/private
          name: systemd-socket

Alertmanager

apiVersion: v1
kind: ConfigMap
metadata:
  name: alertmanager-conf
  namespace: monitoring
data:
  config.yml: |-
    global:
      # 在没有告警的情况下声明为已解决的时间
      resolve_timeout: 1m
      # 配置邮件发送信息
      smtp_smarthost: 'smtp.163.com:25'
      smtp_from: 'sadfds@163.com'
      smtp_auth_username: 'sadfds@163.com'
      smtp_auth_password: 'sadfds'
      # smtp_hello: '163.com'
      smtp_require_tls: false
    # 所有报警信息进入后的根路由，用来设置告警的分发策略
    route:
      # 这里的标签列表是接收到报警信息后的重新分组标签，
      # 例如，接收到的报警信息里面有许多具有 cluster=A 和 alertname=LatncyHigh 
      # 这样的标签的报警信息将会批量被聚合到一个分组里面。
      group_by: ['alertname']

      # 当触发该组的一个告警后，需要等待至少group_wait时间来发送通知，
      # 这段时间内触发的同组告警将一起发送通知，减少告警条数。
      group_wait: 2s

      # 当第一个报警发送后，等待'group_interval'时间后发送该组新告警通知。
      group_interval: 1s

      # 如果一个告警通知已经发送，等待'repeat_interval'时间后再次发送，
      # 减少重复发送条数。
      repeat_interval: 10s

      # 默认的receiver：如果一个告警没有被一个route匹配，则发送给默认的接收器
      receiver: 'sms'

      # 以上属性都由所有子路由继承。
      routes:
      - receiver: 'sms'
        group_wait: 1s
        match:
          team: node
    receivers:
    - name: 'sms'
      webhook_configs:
      - url: 'http://192.168.220.112:5000/'
        send_resolved: false
    # - name: 'email'
    #   email_configs:
    #   - to: '123@qq.com'
    #     send_resolved: true
---        
apiVersion: apps/v1
kind: Deployment
metadata:
  name: alertmanager
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: alertmanager-server
  template:
    metadata:
      labels:
        app: alertmanager-server
    spec:
      nodeSelector:
        kubernetes.io/hostname: knode2
      containers:        
        - name: alermanager
          image: prom/alertmanager:v0.20.0
          imagePullPolicy: IfNotPresent
          args:
          - "--config.file=/etc/alertmanager/config.yml"
          - "--storage.path=/alertmanager/data"
          ports:
          - containerPort: 9093
            name: http
          volumeMounts:
          - mountPath: "/etc/alertmanager"
            name: alertmanager-config-volume
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: 100m
              memory: 256Mi
      volumes:
        - name: alertmanager-config-volume
          configMap:
            name: alertmanager-conf
---
apiVersion: v1
kind: Service
metadata:
  name: alertmanager-service
  namespace: monitoring
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port:   '9093'
spec:
  selector: 
    app: alertmanager-server
  type: NodePort  
  ports:
    - port: 9093
      targetPort: 9093 
      nodePort: 30002
---          
apiVersion: v1
kind: Service
metadata:
  name: alertmanager-inner
  namespace: monitoring
spec:
  clusterIP: None
  ports:
  - name: http
    port: 9093
  selector:
    app: alertmanager-server

说明：

• prometheus.rules 配置中设定的 labels 作为 Alertmanager 的通知路由匹配规则，如team:node，annotations 是告警内容的说明，通常是发送的通知内容。
• Prometheus 的 alert 页面中告警有三种状态:
  • inactive：未触发
  • pending：达到阈值，触发告警，等待 group_wait 和 group_interval超时
  • firing：已将告警发送至 Alertmanager，只要告警没有恢复，就持续在这个状态
• 配置文件里的 send_resolved 我本来理解应该是指告警恢复是否发送通知 ，但实际是指告警恢复后是否仍然发送，所以要配置成 false，否则告警恢复了还会继续发送通知。
• 通过 webhook 接收器自定义通知方式，Alertmanager 会把通知信息 post 到 webhook_configs 配置的url，我们自己启个服务接收通知然后处理，发微信、邮件、钉钉还是短信都可以自己写。
• Alertmanager web界面自带的 silence 功能比较简单，只能根据 label 匹配设置固定的静默时间，通过自定义接收器可以灵活设置静默时间，比如每天的某一时段为部署窗口。

我用 Go 写了一个简单的通知接收程序，将收到的通知打印出来

package main

import (
	"encoding/json"
	"fmt"
	"github.com/julienschmidt/httprouter"
	"io/ioutil"
	"log"
	"net/http"
)

type Msg struct {
	Tel []string `json:"tel"`
	Msg string   `json:"msg"`
}

type Alert struct {
	Labels      map[string]string `json:"labels"`
	Annotations map[string]string `json:"annotations"`
	StartsAt    string            `json:"startsAt"`
	EndsAt      string            `json:"endsAt"`
}

type Alerts struct {
	Version string  ` json:"version"`
	Status  string  `json:"status"`
	Alerts  []Alert `json:"alerts"`
}

func homePage(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reqBody, _ := ioutil.ReadAll(r.Body)
	var alerts Alerts
	err := json.Unmarshal(reqBody, &alerts)
	if err != nil {
		log.Fatal("unmarshal request error: ", err)
	}

	log.Println(alerts)
	_, _ = fmt.Fprintln(w, alerts)
}

func healthCheck(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {

	health := map[string]string{
		"status": "up",
	}
	_ = json.NewEncoder(w).Encode(health)
}

func handleRequests() {
	router := httprouter.New()

	router.POST("/", homePage)
	router.GET("/health", healthCheck)
	log.Fatal(http.ListenAndServe(":5000", router))
}

func main() {
	handleRequests()
}

blackboxexporter_674">blackbox-exporter

apiVersion: apps/v1
kind: Deployment
metadata:
  name: blackbox-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: blackbox-exporter
  template:
    metadata:
      labels:
        app: blackbox-exporter
    spec:
      nodeSelector:
        kubernetes.io/hostname: knode2
      containers:
        - name: blackbox-exporter
          image: prom/blackbox-exporter:v0.16.0
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 9115
            name: http
---
apiVersion: v1
kind: Service
metadata:
  name: blackbox-exporter-inner
  namespace: monitoring
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port:   '9115'
spec:
  selector:
    app: blackbox-exporter
  clusterIP: None
  ports:
  - name: http
    port: 9115