监控常用指标

• node_exporter 标准性能指标

# CPU使用率
100 - (avg(irate(node_cpu_seconds_total{mode='idle', instance="10.12.69.173:9100"}[1m])) *100)
# 内存使用率
100 - (((node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes)/node_memory_MemTotal_bytes) * 100)
# 磁盘使用率
100 - (node_filesystem_free_bytes{fstype=~"ext4|xfs",mountpoint="/",instance="10.12.69.173:9100"} / node_filesystem_size_bytes{fstype=~"ext4|xfs",mountpoint="/",instance="10.12.69.173:9100"} *100)
# 网络传输速率
node_network_receive_bytes_total{device="eth0",instance="10.12.69.173:9100",job="node_exporter"}
# CPU平均负载
node_load5{instance="10.12.69.173:9100",job="node_exporter"}

• mysql 监控性能指标

# mysql存活指标（0代表down，1代表up）
mysql_up
# 每秒查询次数指标
rate(mysql_global_status_slow_queries[5m])
# 连接数指标
rate(mysql_global_status_threads_connected[5m])
# 可用连接数
rate(mysql_global_variables_max_connections - mysql_global_status_threads_connected[5m])
# mysql主从复制I/O线程
mysql_slave_status_slave_sql_running
# mysql主从延迟
rate(mysql_slave_status_seconds_behind_master[5m])

• Pod性能指标

# Pod的cpu使用率
container_memory_usage_bytes{container_name!=""} / container_spec_memory_limit_bytes{container_name!=""}  * 100 != +Inf
# Pod内存使用率
sum by (pod_name)( rate(container_cpu_usage_seconds_total{image!=""}[1m] ) ) * 100

创建规则

Node告警规则

[root@prometheus Prometheus]# mkdir rules
[root@prometheus Prometheus]# vim rules/node_rules.yml
groups:
- name: test
  rules:
  - alert: 内存使用率过高
    expr: 100-(node_memory_Buffers_bytes+node_memory_Cached_bytes+node_memory_MemFree_bytes)/node_memory_MemTotal_bytes*100 > 90
    for: 30s                     # 告警持续时间，超过这个时间才会发送给alertmanager
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} 内存使用率过高"
      description: "{{ $labels.instance }} of job {{$labels.job}}内存使用率超过80%,当前使用率[{{ $value }}]."

  - alert: cpu使用率过高
    expr: 100-avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)*100 > 90
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} cpu使用率过高"
      description: "{{ $labels.instance }} of job {{$labels.job}}cpu使用率超过80%,当前使用率[{{ $value }}]."

MySQL告警规则

• 编写指标

[root@prometheus ~]# vim /usr/local/Prometheus/prometheus/rules/mysql_rules.yml
groups:
- name: mysql_rules
  rules:
  - record: mysql:status
    expr: mysql_up{instance=~".*9104"}
  - record: mysql:uptime
    expr: mysql_global_status_uptime{job="mysqld_exporter"}
  - record: mysql:mysql_threads_connected
    expr: mysql_global_status_threads_connected{job="mysqld_exporter"}
  - record: mysql:mysql_threads_running
    expr: mysql_global_status_threads_running{job="mysqld_exporter"}
  - record: mysql:mysql_aborted_connects
    expr: increase(mysql_global_status_aborted_connects{job="mysqld_exporter"}[2m])
  - record: mysql:mysql_slow_queries
    expr: increase(mysql_global_status_slow_queries{job="mysqld_exporter"}[2m])
  - record: mysql:mysql_table_locks
    expr: increase(mysql_global_status_table_locks_waited{job="mysqld_exporter"}[2m])
  - record: mysql:mysql_qps
    expr: rate(mysql_global_status_queries{job="mysqld_exporter"}[2m])

• 编写告警规则

[root@prometheus ~]# vim /usr/local/Prometheus/prometheus/rules/
groups:
- name: mysql_alerts
  rules:
  - alert: MySQL_Down_Alert
    expr: mysql:status==0
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      summary: 主机 {{ $labels.nodename }} 数据库 出现异常!
      description: 主机 {{ $labels.nodename }} 上的 {{ $labels.job }} 可能存在异常，请检查!
  - alert: MySQL_uptime_Alert
    expr: mysql:uptime<1
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      summary: 主机 {{ $labels.nodename }} 数据库 出现异常!
      description: 主机 {{ $labels.nodename }} 数据库状态异常，请检查!
  - alert: MySQL_threads_connected_Alert
    expr: mysql_threads_connected > 100
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      summary: 主机 {{ $labels.nodename }} 上的数据库指标 threads_connected 超出阈值!
      description: 主机 {{ $labels.nodename }} 上的数据库指标 threads_connected 超出阈值，当前值为{{humanize $value}}，请检查!
  - alert: MySQL_threads_running_Alert
    expr: mysql:mysql_threads_running > 200
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      summary: 主机 {{ $labels.nodename }} 上的数据库指标 threads_running 超出阈值!
      description: 主机 {{ $labels.nodename }} 上的数据库指标 threads_connected 超出阈值，当前值为{{humanize $value}}，请检查!
  - alert: MySQL_aborted_connects_Alert
    expr: mysql:mysql_aborted_connects > 10
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: 
    annotations:
      summary: 主机 {{ $labels.nodename }} 上的数据库指标 aborted_connects 超出阈值
      description: 主机 {{ $labels.nodename }} 上的数据库指标 aborted_connects 超出阈值，当前值为{{humanize $value}}，
  - alert
    expr: mysql:mysql_slow_queries > 1
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      summary: 主机 {{ $labels.nodename }} 上的数据库指标 slow_queries 超出阈值!
      description: 主机 {{ $labels.nodename }} 上的数据库指标 slow_queries 超出阈值，当前值为{{humanize $value}}，请检查!
  - alert: MySQL_table_locks_Alert
    expr: mysql:mysql_table_locks > 1
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      ummary: 主机 {{ $labels.nodename }} 上的数据库指标 table_locks 超出阈值
      description: 主机 {{ $labels.nodename }} 上的数据库指标 table_locks 超出阈值，当前值为{{humanize $value}}，请检查!
  - alert: MySQL_qps_Alert
    expr: mysql:mysql_qps > 500
    for: 1m
    labels:
      metric_type: db_monitor
      resource: db
      severity: critical
    annotations:
      summary: 主机 {{ $labels.nodename }} 上的数据库指标 qps 超出阈值!
      description: 主机 {{ $labels.nodename }} 上的数据库指标 qps 超出阈值，当前值为{{humanize $value}}，请检查!

Pod告警规则

groups:
- name: noah_pod.rules
  rules:
  - alert: PodMemUsage
    expr: container_memory_usage_bytes{container_name!=""} / container_spec_memory_limit_bytes{container_name!=""}  *100 != +Inf > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.name}}: Pod High Mem usage detected"
      description: "{{$labels.name}}: Pod Mem is above 80% ,(current value is: {{ $value }})"
  - alert: PodCpuUsage
    expr: sum by (pod_name)( rate(container_cpu_usage_seconds_total{image!=""}[1m] ) ) * 100 > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.name}}: Pod High CPU usage detected"
      description: "{{$labels.name}}: Pod CPU is above 80% ,(current value is: {{ $value }})"