harbor version: 2.10.0
Harbor 监控指标主要包含以下几个部分，具体可以参考 Harbor Exporter Metrics
- harbor health
- system info
- project info
- job service
harbor health
harbor health 通过调用 /api/v2.0/health 查询，代码如下：
~/go/src/xxx/harborctl (master ✗) curl -k -u 'user@password' -X GET https://reg.srv.xxx.cn/api/v2.0/health | jq
{
"components": [
{
"name": "core",
"status": "healthy"
},
{
"name": "database",
"status": "healthy"
},
{
"name": "jobservice",
"status": "healthy"
},
{
"name": "portal",
"status": "healthy"
},
{
"name": "redis",
"status": "healthy"
},
{
"name": "registry",
"status": "healthy"
},
{
"name": "registryctl",
"status": "healthy"
}
],
"status": "healthy"
}
System info
system info 通过调用 /api/v2.0/systeminfo查询，代码如下：
~/go/src/xxx/harborctl (master ✗) curl -k -u 'user@password' -X GET https://reg.srv.xxx.cn/api/v2.0/systeminfo | jq
{
"auth_mode": "ldap_auth",
"banner_message": "",
"current_time": "2024-03-12T03:54:47.921Z",
"external_url": "https://reg.srv.xxx.cn",
"harbor_version": "v2.10.0-6abb4eab",
"has_ca_root": false,
"notification_enable": true,
"oidc_provider_name": "",
"primary_auth_mode": false,
"project_creation_restriction": "adminonly",
"read_only": false,
"registry_storage_provider_name": "s3",
"registry_url": "reg.srv.xxx.cn",
"self_registration": false
}
Project info
Project info 通过读取数据库记录进行查询，代码如下：
harbor=> SELECT project_metadata.value AS public, COUNT(project_metadata.value) AS count
harbor-> FROM project INNER JOIN project_metadata ON project.project_id=project_metadata.project_id
harbor-> WHERE project.deleted=FALSE AND project_metadata.name='public'
harbor-> GROUP BY project_metadata.value;
public | count
--------+-------
true | 19
false | 12
(2 rows)
harbor=> SELECT project.project_id, project.name, project_metadata.value AS public, quota.hard AS quota, quota_usage.used AS usage FROM project
harbor-> INNER JOIN project_metadata ON (project.project_id = project_metadata.project_id)
harbor-> INNER JOIN quota ON project.project_id = CAST(quota.reference_id AS Integer)
harbor-> INNER JOIN quota_usage ON project.project_id = CAST(quota_usage.reference_id AS Integer)
harbor-> WHERE quota.reference='project' AND quota_usage.reference='project' AND project.deleted=FALSE AND project_metadata.name='public';
project_id | name | public | quota | usage
------------+-------------------------------+--------+---------------------------+-----------------------------
28 | xxx-prediction | false | {"storage": -1} | {"storage": 66540139800}
19 | hivesec | true | {"storage": -1} | {"storage": 29397564588}
23 | kubeedge-cd | false | {"storage": -1} | {"storage": 27229995516}
69 | xxx-onboard-system | true | {"storage": -1} | {"storage": 15794403147}
18 | infra-operation | false | {"storage": -1} | {"storage": 1086170972483}
29 | xxx-common-test | true | {"storage": 536870912000} | {"storage": 10326108416}
3 | xxx-all | true | {"storage": -1} | {"storage": 5141042260695}
4 | xxx-ai-platform | true | {"storage": -1} | {"storage": 33540785742}
17 | xxx-test | true | {"storage": -1} | {"storage": 60594336086}
16 | perception-test | true | {"storage": -1} | {"storage": 1281251136969}
67 | cloud-test | true | {"storage": -1} | {"storage": 35962611310}
11 | xxx-simulation-services | true | {"storage": -1} | {"storage": 962089168777}
34 | xxx-edge | true | {"storage": -1} | {"storage": 396728976}
8 | xxx-production | false | {"storage": -1} | {"storage": 778171807755}
32 | simulation-jobs | true | {"storage": -1} | {"storage": 7500526904231}
68 | xxx-tke | false | {"storage": -1} | {"storage": 24142197231}
5 | xxx-development | false | {"storage": -1} | {"storage": 865725642635}
24 | xxx-modules-all | true | {"storage": -1} | {"storage": 28905956667895}
1 | library | true | {"storage": -1} | {"storage": 35962611310}
6 | xxx-infra | false | {"storage": -1} | {"storage": 1599663489}
14 | xxx-uat | false | {"storage": -1} | {"storage": 19526673706}
25 | xxx-ui | true | {"storage": -1} | {"storage": 150635978}
10 | xxx-simulation | true | {"storage": -1} | {"storage": 7349189814197}
21 | xxx-rd-testing | false | {"storage": -1} | {"storage": 1899226810}
31 | data-closedloop | true | {"storage": -1} | {"storage": 20697007871}
27 | xxx-production-external | false | {"storage": -1} | {"storage": 233172664}
15 | google_containers | true | {"storage": -1} | {"storage": 235452327}
9 | xxx-public | true | {"storage": -1} | {"storage": 1606672360903}
12 | xxx-staging | false | {"storage": -1} | {"storage": 607922059044}
13 | xxx-training-framework | true | {"storage": -1} | {"storage": 22876361554}
7 | xxx-perception | false | {"storage": -1} | {"storage": 749309031}
(31 rows)
harbor=> SELECT project.project_id, COUNT(project.project_id) AS member_total
harbor-> FROM project INNER JOIN project_member ON project.project_id=project_member.project_id
harbor-> WHERE project.deleted=FALSE AND project_member.entity_type='u'
harbor-> GROUP BY project.project_id, project_member.entity_type;
project_id | member_total
------------+--------------
11 | 6
1 | 2
5 | 10
12 | 10
68 | 2
10 | 2
19 | 3
27 | 2
21 | 4
23 | 9
4 | 3
29 | 6
14 | 2
3 | 40
24 | 7
32 | 4
6 | 2
15 | 2
28 | 5
9 | 21
7 | 2
8 | 9
34 | 3
18 | 10
31 | 2
17 | 10
13 | 4
69 | 4
67 | 1
25 | 4
16 | 11
(31 rows)
harbor=> SELECT repository.project_id, COUNT(repository.project_id) AS repo_total, SUM(repository.pull_count) AS pull_total
harbor-> FROM project INNER JOIN repository ON project.project_id=repository.project_id
harbor-> WHERE project.deleted=FALSE
harbor-> GROUP BY repository.project_id;
project_id | repo_total | pull_total
------------+------------+------------
29 | 5 | 0
68 | 9 | 0
4 | 68 | 0
34 | 1 | 0
67 | 4 | 5
32 | 2 | 0
10 | 27 | 0
7 | 1 | 0
9 | 1446 | 0
15 | 7 | 0
6 | 9 | 0
12 | 147 | 0
24 | 8730 | 0
19 | 84 | 0
25 | 3 | 0
69 | 3 | 0
31 | 12 | 0
21 | 2 | 0
14 | 17 | 0
3 | 522 | 0
17 | 76 | 0
28 | 6 | 0
13 | 1 | 0
1 | 4 | 0
5 | 201 | 2
18 | 2054 | 0
16 | 206 | 0
27 | 6 | 0
23 | 52 | 0
11 | 107 | 0
8 | 215 | 0
(31 rows)
harbor=> SELECT artifact.project_id, artifact.type AS artifact_type, COUNT(artifact.type) AS artifact_total
harbor-> FROM project INNER JOIN artifact ON project.project_id=artifact.project_id
harbor-> WHERE project.deleted=FALSE
harbor-> GROUP BY artifact.project_id, type;
project_id | artifact_type | artifact_total
------------+---------------+----------------
1 | IMAGE | 43
3 | IMAGE | 8915
4 | IMAGE | 80
5 | IMAGE | 11017
6 | IMAGE | 47
7 | IMAGE | 1
8 | IMAGE | 7276
9 | IMAGE | 12395
9 | UNKNOWN | 1
10 | IMAGE | 2358
11 | IMAGE | 12426
12 | IMAGE | 6912
13 | IMAGE | 8
14 | IMAGE | 900
15 | IMAGE | 8
16 | IMAGE | 964
17 | IMAGE | 364
18 | CHART | 2
18 | IMAGE | 13867
19 | IMAGE | 102
21 | IMAGE | 4
23 | IMAGE | 222
24 | IMAGE | 40259
25 | IMAGE | 44
27 | IMAGE | 56
28 | IMAGE | 18
29 | IMAGE | 6
31 | IMAGE | 35
32 | IMAGE | 3221
34 | IMAGE | 52
67 | IMAGE | 43
68 | IMAGE | 66
69 | IMAGE | 20
(33 rows)
Job service
以下几个 task 相关的 metrics 是通过 harbor exporter 组件进行获取的。
task_queue_size, task_queue_latency
r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com:6379[1]> LLEN "{harbor-server}:jobs:GARBAGE_COLLECTION"
(integer) 0
r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com:6379[1]> LINDEX "{harbor-server}:jobs:GARBAGE_COLLECTION" -1
(nil)
task_concurrency
🍺 /home/xxx/liangzheng ☞ redis-cli -h r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com -a <password>
r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com:6379> SELECT 1
OK
r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com:6379[1]> SMEMBERS "{harbor-server}:known_jobs"
1) "GARBAGE_COLLECTION"
2) "REPLICATION"
3) "IMAGE_SCAN"
4) "IMAGE_SCAN_ALL"
5) "WEBHOOK"
6) "SCAN_DATA_EXPORT"
7) "SCHEDULER"
8) "PURGE_AUDIT_LOG"
9) "DEMO"
10) "EXECUTION_SWEEP"
11) "IMAGE_GC"
12) "RETENTION"
13) "IMAGE_REPLICATE"
14) "SLACK"
15) "SYSTEM_ARTIFACT_CLEANUP"
16) "AUDIT_LOGS_GDPR_COMPLIANT"
17) "P2P_PREHEAT"
r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com:6379[1]> HGETALL "{harbor-server}:jobs:REPLICATION:lock_info"
1) "df1fa66ce3992cc9e1449483"
2) "0"
3) "1ef8a49c990d15220fd5443c"
4) "0"
5) "3280a0bc80141409ac2d9e27"
6) "0"
task_scheduled_total
r-wz9m4ifqyj1opzlies.redis.rds.aliyuncs.com:6379[1]> ZRANGEBYSCORE "{harbor-server}:scheduled" -inf +inf withscores limit 0 20
(empty list or set)
以下三个参数是通过 Harbor Jobservice 组件获取的，代码如下：
jobservice_info
Jobservice info 包括：
- node 信息
- redis pool id
- redis worker count，这个值首先会从环境变量
JOB_SERVICE_POOL_WORKERS中读取，如果不设置则默认为 10
# HELP jobservice_info the information of jobservice
# TYPE jobservice_info gauge
jobservice_info{node="harbor-jobservice-64644b6477-d4pl5:10.233.107.26",pool="2ca124a2f99043abecdd4ba9",workers="10"} 1
jobservice_task_process_time_seconds 和 jobservice_task_total
各种 type（例如 EXECUTION_SWEEP, SCHEDULER 等等） 的 job （fail, stop, success）运行时间和状态统计
# HELP jobservice_task_process_time_seconds The time duration of the task processing time
# TYPE jobservice_task_process_time_seconds summary
jobservice_task_process_time_seconds{status="success",type="EXECUTION_SWEEP",quantile="0.5"} NaN
jobservice_task_process_time_seconds{status="success",type="EXECUTION_SWEEP",quantile="0.9"} NaN
jobservice_task_process_time_seconds{status="success",type="EXECUTION_SWEEP",quantile="0.99"} NaN
jobservice_task_process_time_seconds_sum{status="success",type="EXECUTION_SWEEP"} 0.512839206
jobservice_task_process_time_seconds_count{status="success",type="EXECUTION_SWEEP"} 1
# HELP jobservice_task_total The number of processed tasks
# TYPE jobservice_task_total counter
jobservice_task_total{status="success",type="EXECUTION_SWEEP"} 1
jobservice_task_total{status="success",type="SCHEDULER"} 1
参考链接
https://github.com/gocraft/work
https://goharbor.io/docs/2.10.0/administration/jobservice-dashboard/
