M3DB

分别配置Coordinator和dbnode

Coordinator配置文件

Coordinator是在prometheus侧负责将数据读写到DBnode，Coordinator本身不存储任何数据

listenAddress:
  type: "config"
  value: "0.0.0.0:7201"  # 交互端口

metrics:                  # coordinator本身的metric
  scope:
    prefix: "coordinator"
  prometheus:
    handlerPath: /monitor/metrics
    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
  sanitization: prometheus
  samplingRate: 1.0
  extended: none

limits:
  maxComputedDatapoints: 10000

tagOptions:
  # Configuration setting for generating metric IDs from tags.
  idScheme: quoted

clusters:
## Fill-out the following and un-comment before using, and
### make sure indent by two spaces is applied.
  - namespaces:
      - namespace: metrics
        retention: 48h
        type: unaggregated
    client:
      config:
        service:
          env: default_env
          zone: embedded
          service: m3db
          cacheDir: /var/lib/m3kv
          etcdClusters:
            - zone: embedded
              endpoints:
                - HOST1_STATIC_IP_ADDRESS:2379
                - HOST2_STATIC_IP_ADDRESS:2379
                - HOST3_STATIC_IP_ADDRESS:2379
##                - M3DB_NODE_02_STATIC_IP_ADDRESS:2379
##                - M3DB_NODE_03_STATIC_IP_ADDRESS:2379
###               ... etc, list only M3DB seed nodes
      writeConsistencyLevel: majority
      readConsistencyLevel: unstrict_majority
      writeTimeout: 10s
      fetchTimeout: 15s
      connectTimeout: 20s
      writeRetry:
        initialBackoff: 500ms
        backoffFactor: 3
        maxRetries: 2
        jitter: true
      fetchRetry:
        initialBackoff: 500ms
        backoffFactor: 2
        maxRetries: 3
        jitter: true
      backgroundHealthCheckFailLimit: 4
      backgroundHealthCheckFailThrottleFactor: 0.5

启动方式：m3coordinator -f <config-name.yml>

dbnode配置文件

db:
  logging:
    level: info

  metrics:
    prometheus:
      handlerPath: /monitor/metrics
    sanitization: prometheus
    samplingRate: 1.0
    extended: detailed

  hostID:                   # 可以理解为nodeID
    resolver: config        # 单机多实例，这里必须设置为config，从文件中读取
    value:  151        # 这个实例的ID
    
  config:
    service:
        env: default_env
        zone: embedded
        service: m3db    # 服务名，可以按照consul中的service进行理解
        cacheDir: /var/lib/m3kv
        etcdClusters:
          - zone: embedded
            endpoints:
              - HOST1_STATIC_IP_ADDRESS:2379
              - HOST2_STATIC_IP_ADDRESS:2379
              - HOST3_STATIC_IP_ADDRESS:2379
    seedNodes:
      initialCluster:
        - hostID: HOST1_NAME
          endpoint: http://HOST1_STATIC_IP_ADDRESS:2380
        - hostID: HOST2_NAME
          endpoint: http://HOST2_STATIC_IP_ADDRESS:2380
        - hostID: HOST2_NAME
          endpoint: http://HOST3_STATIC_IP_ADDRESS:2380

  listenAddress: 0.0.0.0:9000
  clusterListenAddress: 0.0.0.0:9001
  httpNodeListenAddress: 0.0.0.0:9002
  httpClusterListenAddress: 0.0.0.0:9003
  debugListenAddress: 0.0.0.0:9004

  client:
    writeConsistencyLevel: majority           # 写一致性级别
    readConsistencyLevel: unstrict_majority   # 读一致性级别

  gcPercentage: 100

  writeNewSeriesAsync: true
  writeNewSeriesLimitPerSecond: 1048576
  writeNewSeriesBackoffDuration: 2ms

  bootstrap:          # 启动顺序
    bootstrappers:
        - filesystem
        - commitlog
        - peers
        - uninitialized_topology
    commitlog:
      returnUnfulfilledForCorruptCommitLogFiles: false

  cache:
    series:
      policy: lru
    postingsList:
      size: 262144

  commitlog:
    flushMaxBytes: 524288
    flushEvery: 1s
    queue:
      calculationType: fixed
      size: 2097152
  fs:
    filePathPrefix: /var/lib/m3db    # m3dbnode数据目录

  # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/
  # for quick local setup (which this config will send data to).

  # tracing:
  #  backend: jaeger

启动方式：m3dbnode -f <config-name.yml>

部分报错解决

current value for RLIMIT_NOFILE(1048576) is below recommended threshold(3000000)\nmax value for RLIMIT_NOFILE(1048576) is below recommended threshold(3000000)

临时解决：

ulimit -n 3000000

永久修改：

vim /etc/systemd/system.conf
DefaultLimiterNOFILE=3000000

重启机器生效

current value for vm.max_map_count(3000) is below recommended threshold(3000000)"

临时修改：

sysctl -w vm.max_map_count=3000000
sysctl -w vm.swappiness=1
sysctl -w fs.file-max=3000000
sysctl -w fs.nr_open=3000000

永久修改：

vim /etc/sysctl.conf

vm.max_map_count = 3000000
vm.swappiness = 1
fs.file-max = 3000000
fs.nr_open = 3000000

sysctl -p

初始化

id需要与配置文件中已知，endpoint填写节点ip及端口号，hostname为机器全名，使用hostname -f查看，isolation_group必须大于等于replication_factor。

curl -X POST localhost:7201/api/v1/placement/init -d '{
    "num_shards": 1024,
    "replication_factor": 3,
    "instances": [
        {
            "id": "151",
            "isolation_group": "us-east-a",
            "zone": "embedded",
            "weight": 100,
            "endpoint": "HOST1_STATIC_IP_ADDRESS:9000",
            "hostname": "Host1Name",
            "port": 9000
        },
        {
            "id": "153",
            "isolation_group": "us-east-b",
            "zone": "embedded",
            "weight": 100,
            "endpoint": "HOST1_STATIC_IP_ADDRESS:9000",
            "hostname": "Host2Name",
            "port": 9000
        },
        {
            "id": "154",
            "isolation_group": "us-east-c",
            "zone": "embedded",
            "weight": 100,
            "endpoint": "HOST1_STATIC_IP_ADDRESS:9000",
            "hostname": "Host3Name",
            "port": 9000
        }
    ]
}'

测试

写一个metrics：

curl -sS -X POST localhost:9003/writetagged -d '{
  "namespace": "metrics",
  "id": "foo",
  "tags": [
    {
      "name": "city",
      "value": "new_york"
    },
    {
      "name": "endpoint",
      "value": "/request"
    }
  ],
  "datapoint": {
    "timestamp": '"$(date "+%s")"',
    "value": 42.123456789
  }
}'

然后读取刚刚写入的metrics：

curl -sS -X POST http://localhost:9003/query -d '{
  "namespace": "metrics",
  "query": {
    "regexp": {
      "field": "city",
      "regexp": ".*"
    }
  },
  "rangeStart": 0,
  "rangeEnd": '"$(date "+%s")"'
}'

结果：

{
    "results": [{
        "id": "foo",
        "tags": [{
            "name": "city",
            "value": "new_york"
        }, {
            "name": "endpoint",
            "value": "/request"
        }],
        "datapoints": [{
            "timestamp": 1566267395,
            "value": 42.123456789
        }, {
            "timestamp": 1566267610,
            "value": 42.123456789
        }, {
            "timestamp": 1566286208,
            "value": 42.123456789
        }, {
            "timestamp": 1566354670,
            "value": 42.123456789
        }, {
            "timestamp": 1566790936,
            "value": 42.123456789
        }, {
            "timestamp": 1566790945,
            "value": 42.123456789
        }]
    }],
    "exhaustive": true
}