image.png
otel配置文件：
exporters:
clickhouse:
endpoint: tcp://192.167.0.250:9000
database: otel
logs_table_name: otel_logs
traces_table_name: otel_traces
ttl_days: 3
timeout: 10s
sending_queue:
queue_size: 100
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s
1、 登录clickhouse，创建otel库
2、 在demo应用启动vm参数中加上 ：
-javaagent:D:\git\opensource\opentelemetry\opentelemetry-javaagent.jar
3、启动otel collector：
otelcontribcol.exe --config=config-ck.yml
注意要使用社区版otelcontribcol，正式版没有这个功能
启动后我们看到CK上已经有调用链的表了
建表语句：
具体可以参见
CREATE TABLE IF NOT EXISTS %s (
Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
TraceId String CODEC(ZSTD(1)),
SpanId String CODEC(ZSTD(1)),
ParentSpanId String CODEC(ZSTD(1)),
TraceState String CODEC(ZSTD(1)),
SpanName LowCardinality(String) CODEC(ZSTD(1)),
SpanKind LowCardinality(String) CODEC(ZSTD(1)),
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
ScopeName String CODEC(ZSTD(1)),
ScopeVersion String CODEC(ZSTD(1)),
SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
Duration Int64 CODEC(ZSTD(1)),
StatusCode LowCardinality(String) CODEC(ZSTD(1)),
StatusMessage String CODEC(ZSTD(1)),
Events Nested (
Timestamp DateTime64(9),
Name LowCardinality(String),
Attributes Map(LowCardinality(String), String)
) CODEC(ZSTD(1)),
Links Nested (
TraceId String,
SpanId String,
TraceState String,
Attributes Map(LowCardinality(String), String)
) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_duration Duration TYPE minmax GRANULARITY 1
) ENGINE MergeTree()
%s
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId)
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
4、安装grafana的clickhouse插件
[https://grafana.com/grafana/plugins/grafana-clickhouse-datasource/]
安装后增加一个datasource
在explorer中可以查看数据
5、 导入trace的dashboard
trace.json
{
"__inputs": [
{
"name": "DS_CLICKHOUSE",
"label": "ClickHouse",
"description": "",
"type": "datasource",
"pluginId": "grafana-clickhouse-datasource",
"pluginName": "ClickHouse"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.4.1"
},
{
"type": "datasource",
"id": "grafana-clickhouse-datasource",
"name": "ClickHouse",
"version": "2.1.1"
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
},
{
"type": "panel",
"id": "traces",
"name": "Traces",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 0,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"maxDataPoints": 50,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"groupBy": [
"ServiceName"
],
"limit": 10000,
"metrics": [
{
"aggregation": "count",
"field": ""
}
],
"mode": "trend",
"orderBy": [],
"table": "otel_traces",
"timeField": "Timestamp",
"timeFieldType": "DateTime64(9)"
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"format": 0,
"meta": {
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"groupBy": [
"ServiceName"
],
"limit": 10000,
"metrics": [
{
"aggregation": "count",
"field": ""
}
],
"mode": "trend",
"orderBy": [],
"table": "otel_traces",
"timeField": "Timestamp",
"timeFieldType": "DateTime64(9)"
}
},
"queryType": "sql",
"rawSql": "SELECT $__timeInterval(Timestamp) as time, ServiceName, count() as ` ` FROM otel.\"otel_traces\" WHERE $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id) AND $__timeFilter(Timestamp) AND ServiceName In (${serviceName:singlequote}) GROUP BY ServiceName, time ORDER BY time ASC LIMIT 100000",
"refId": "A",
"selectedFormat": 0
}
],
"title": "Services over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 9
},
"id": 7,
"maxDataPoints": 50,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"groupBy": [
"ServiceName"
],
"limit": 10000,
"metrics": [
{
"aggregation": "count",
"field": ""
}
],
"mode": "trend",
"orderBy": [],
"table": "otel_traces",
"timeField": "Timestamp",
"timeFieldType": "DateTime64(9)"
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"format": 0,
"meta": {
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"groupBy": [
"ServiceName"
],
"limit": 10000,
"metrics": [
{
"aggregation": "count",
"field": ""
}
],
"mode": "trend",
"orderBy": [],
"table": "otel_traces",
"timeField": "Timestamp",
"timeFieldType": "DateTime64(9)"
}
},
"queryType": "sql",
"rawSql": "SELECT $__timeInterval(Timestamp) as time, ServiceName,\n\tquantile(0.99)(Duration)/1000000 AS p99 FROM otel.\"otel_traces\" WHERE $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id) AND $__timeFilter(Timestamp) AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime ) AND ServiceName In (${serviceName:singlequote}) AND ServiceName != 'loadgenerator' GROUP BY time, ServiceName ORDER BY time ASC LIMIT 100000",
"refId": "A",
"selectedFormat": 0
}
],
"title": "Service Performance - p99",
"type": "timeseries"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
"fillOpacity": 24,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 18
},
"id": 8,
"maxDataPoints": 50,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.4.1",
"targets": [
{
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"groupBy": [
"ServiceName"
],
"limit": 10000,
"metrics": [
{
"aggregation": "count",
"field": ""
}
],
"mode": "trend",
"orderBy": [],
"table": "otel_traces",
"timeField": "Timestamp",
"timeFieldType": "DateTime64(9)"
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"format": 0,
"meta": {
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"groupBy": [
"ServiceName"
],
"limit": 10000,
"metrics": [
{
"aggregation": "count",
"field": ""
}
],
"mode": "trend",
"orderBy": [],
"table": "otel_traces",
"timeField": "Timestamp",
"timeFieldType": "DateTime64(9)"
}
},
"queryType": "sql",
"rawSql": "SELECT $__timeInterval(Timestamp) as time, count(*) as ` `,\n\tServiceName FROM otel.\"otel_traces\" WHERE $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id) AND $__timeFilter(Timestamp) AND ServiceName In (${serviceName:singlequote}) AND StatusCode= 'STATUS_CODE_ERROR' and ServiceName != 'loadgenerator' GROUP BY ServiceName, time ORDER BY time ASC LIMIT 100000",
"refId": "A",
"selectedFormat": 0
}
],
"title": "Error rates",
"type": "timeseries"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Trace Id"
},
"properties": [
{
"id": "custom.width",
"value": 207
},
{
"id": "links",
"value": [
{
"title": "__value.raw",
"url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${serviceName:queryparam}&var-trace_id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "Service Name"
},
"properties": [
{
"id": "custom.width",
"value": 184
},
{
"id": "links",
"value": [
{
"title": "__value.raw",
"url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "Duration"
},
"properties": [
{
"id": "custom.width",
"value": 363
},
{
"id": "unit",
"value": "ms"
},
{
"id": "custom.cellOptions",
"value": {
"mode": "basic",
"type": "gauge"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Timestamp"
},
"properties": [
{
"id": "custom.width",
"value": 216
}
]
},
{
"matcher": {
"id": "byName",
"options": "Service Tags"
},
"properties": [
{
"id": "custom.inspect",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "min(Timestamp)"
},
"properties": [
{
"id": "custom.width",
"value": 248
}
]
},
{
"matcher": {
"id": "byName",
"options": "timestamp"
},
"properties": [
{
"id": "custom.width",
"value": 234
}
]
}
]
},
"gridPos": {
"h": 15,
"w": 12,
"x": 0,
"y": 27
},
"id": 4,
"options": {
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "timestamp"
}
]
},
"pluginVersion": "9.4.1",
"targets": [
{
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"limit": 100,
"mode": "list",
"orderBy": [],
"table": "otel_traces"
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"format": 1,
"meta": {
"builderOptions": {
"database": "otel",
"fields": [],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "Timestamp",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "Timestamp",
"name": "Timestamp",
"picklistValues": [],
"type": "DateTime64(9)"
}
],
"type": "datetime"
}
],
"limit": 100,
"mode": "list",
"orderBy": [],
"table": "otel_traces"
}
},
"queryType": "sql",
"rawSql": "SELECT min(Timestamp) as timestamp, TraceId as `Trace Id`, argMin(ServiceName, Timestamp) as `Service Name`, sum(Duration)/1000000 as Duration from otel.otel_traces WHERE $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id) AND ServiceName In (${serviceName:singlequote}) AND ServiceName != 'loadgenerator' AND $__timeFilter(Timestamp) GROUP BY TraceId ORDER BY Duration DESC LIMIT 100",
"refId": "A",
"selectedFormat": 1
}
],
"title": "Traces",
"type": "table"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"gridPos": {
"h": 15,
"w": 12,
"x": 12,
"y": 27
},
"id": 6,
"targets": [
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"format": 3,
"meta": {
"builderOptions": {
"fields": [],
"limit": 100,
"mode": "list"
}
},
"queryType": "sql",
"rawSql": "SELECT TraceId as traceID, SpanId as spanID, SpanName as operationName, ParentSpanId as parentSpanID, ServiceName as serviceName, Duration/1000000 as duration, Timestamp as startTime, arrayMap(key -> map('key', key, 'value',SpanAttributes[key]), mapKeys(SpanAttributes)) as tags, arrayMap(key -> map('key', key, 'value',ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags FROM otel.otel_traces WHERE TraceId = '${trace_id}' ORDER BY startTime ASC",
"refId": "A",
"selectedFormat": 3
}
],
"title": "Trace Details",
"type": "traces"
}
],
"refresh": "",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces",
"hide": 0,
"includeAll": true,
"label": "Service Name",
"multi": true,
"name": "serviceName",
"options": [],
"query": "SELECT DISTINCT ServiceName FROM otel.otel_traces",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"allValue": "empty",
"current": {},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "${DS_CLICKHOUSE}"
},
"definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100",
"hide": 0,
"includeAll": true,
"label": "Trace Id",
"multi": false,
"name": "trace_id",
"options": [],
"query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "OTEL Traces",
"uid": "8klBUGfVk",
"version": 11,
"weekStart": ""
}
有的地方说安装clickhouse插件后就自动有trace查看能力，我试了一下并没有，还是要导入一个dashboard
这个dashboard提供了基本的trace查看能力
