1. 部署elasticsearch-exporter进行指标抓取
vim es-exporter.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: es-exporter
namespace: monitoring
labels:
app: es-exporter
spec:
replicas: 1
selector:
matchLabels:
app: es-exporter
template:
metadata:
labels:
app: es-exporter
spec:
containers:
- name: es-exporter
image: bitnami/elasticsearch-exporter:1.6.0
ports:
- containerPort: 9114
args:
- --es.uri=http://10.1.2.232:9200 #es地址
resources:
requests:
memory: "100Mi"
cpu: "100m"
limits:
memory: "200Mi"
cpu: "200m"
---
apiVersion: v1
kind: Service
metadata:
name: es-exporter
namespace: monitoring
labels:
app: es-exporter
spec:
ports:
- name: metrics
port: 9114
targetPort: 9114
protocol: TCP
selector:
app: es-exporter
type: ClusterIP
可以看到部署成功
$ kubectl get po -n monitoring |grep es
es-exporter-7fddbb7b9b-ltj68 1/1 Running 0 48m
$ kubectl get svc -n monitoring |grep es
es-exporter ClusterIP 10.105.28.44 <none> 9114/TCP 46m
- 测试抓取数据：
curl http://10.105.28.44:9114/metrics
2. 配置ServiceMonitor和prometheusrules
vim es-ServiceMonitor-prometheusrules.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: es-exporter
namespace: monitoring
labels:
release: prometheus # 替换为你的 Prometheus Operator 的 release 标签值
spec:
selector:
matchLabels:
app: es-exporter
namespaceSelector:
matchNames:
- monitoring # 替换为部署 es-exporter 的命名空间
endpoints:
- port: metrics
interval: 15s
scheme: http
path: /metrics
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: es-rule
namespace: monitoring
spec:
groups:
- name: elasticsearch
rules:
- alert: ElasticsearchHeapUsageTooHigh
expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90
for: 2m
labels:
severity: critical
annotations:
summary: Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})
description: "Elasticsearch 堆内存使用超过 90%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchHeapUsageWarning
expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80
for: 2m
labels:
severity: warning
annotations:
summary: Elasticsearch Heap Usage warning (instance {{ $labels.instance }})
description: "Elasticsearch 堆内存使用超过 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchDiskOutOfSpace
expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10
for: 0m
labels:
severity: critical
annotations:
summary: Elasticsearch disk out of space (instance {{ $labels.instance }})
description: "Elasticsearch 磁盘空间不足，剩余空间小于 10%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchDiskSpaceLow
expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20
for: 2m
labels:
severity: warning
annotations:
summary: Elasticsearch disk space low (instance {{ $labels.instance }})
description: "Elasticsearch 磁盘空间较低，剩余空间小于 20%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchClusterRed
expr: elasticsearch_cluster_health_status{color="red"} == 1
for: 0m
labels:
severity: critical
annotations:
summary: Elasticsearch Cluster Red (instance {{ $labels.instance }})
description: "Elasticsearch 集群状态为红色（RED），需要立即处理\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchClusterYellow
expr: elasticsearch_cluster_health_status{color="yellow"} == 1
for: 0m
labels:
severity: warning
annotations:
summary: Elasticsearch Cluster Yellow (instance {{ $labels.instance }})
description: "Elasticsearch 集群状态为黄色（YELLOW），可能存在部分问题\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchHealthyNodes
expr: elasticsearch_cluster_health_number_of_nodes < 3
for: 0m
labels:
severity: info
annotations:
summary: Elasticsearch Healthy Nodes (instance {{ $labels.instance }})
description: "Elasticsearch 集群中健康节点数量少于 3 个\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchHealthyDataNodes
expr: elasticsearch_cluster_health_number_of_data_nodes < 3
for: 0m
labels:
severity: info
annotations:
summary: Elasticsearch Healthy Data Nodes (instance {{ $labels.instance }})
description: "Elasticsearch 集群中健康数据节点数量少于 3 个\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchRelocatingShards
expr: elasticsearch_cluster_health_relocating_shards > 0
for: 0m
labels:
severity: info
annotations:
summary: Elasticsearch relocating shards (instance {{ $labels.instance }})
description: "Elasticsearch 正在迁移分片\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchRelocatingShardsTooLong
expr: elasticsearch_cluster_health_relocating_shards > 0
for: 15m
labels:
severity: warning
annotations:
summary: Elasticsearch relocating shards too long (instance {{ $labels.instance }})
description: "Elasticsearch 已经迁移分片超过 15 分钟\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchInitializingShards
expr: elasticsearch_cluster_health_initializing_shards > 0
for: 0m
labels:
severity: info
annotations:
summary: Elasticsearch initializing shards (instance {{ $labels.instance }})
description: "Elasticsearch 正在初始化分片\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchInitializingShardsTooLong
expr: elasticsearch_cluster_health_initializing_shards > 0
for: 15m
labels:
severity: warning
annotations:
summary: Elasticsearch initializing shards too long (instance {{ $labels.instance }})
description: "Elasticsearch 初始化分片时间超过 15 分钟\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchUnassignedShards
expr: elasticsearch_cluster_health_unassigned_shards > 0
for: 0m
labels:
severity: critical
annotations:
summary: Elasticsearch unassigned shards (instance {{ $labels.instance }})
description: "Elasticsearch 存在未分配的分片\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchPendingTasks
expr: elasticsearch_cluster_health_number_of_pending_tasks > 0
for: 15m
labels:
severity: warning
annotations:
summary: Elasticsearch pending tasks (instance {{ $labels.instance }})
description: "Elasticsearch 存在待处理任务，集群性能可能受到影响\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ElasticsearchNoNewDocuments
expr: increase(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1
for: 0m
labels:
severity: warning
annotations:
summary: Elasticsearch no new documents (instance {{ $labels.instance }})
description: "Elasticsearch 过去 10 分钟内没有新增文档\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- 可以看到成功监控，告警规则也有了
image.png
image.png
