docker-compose部署zookeeper&kafka集群

环境说明

Zookeeper集群

IP	节点目录名	容器名	client port	follower port	election port
192.168.78.155	zookeeper1	zookeeper1	2181	2888	3888
192.168.78.156	zookeeper2	zookeeper2	2181	2888	3888
192.168.78.157	zookeeper3	zookeeper3	2181	2888	3888

Kafka集群

IP	节点目录名	容器名	占用端口
192.168.78.155	kafka1	kafka1	9092
192.168.78.156	kafka2	kafka2	9092
192.168.78.157	kafka3	kafka3	9092
192.168.78.155	kafka-manager	kafka-manager	9000

每台服务器上安装docker-compose

[root@kafka1 ~]# sudo curl -L "https://github.com/docker/compose/releases/download/1.27.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
[root@kafka1 ~]# sudo chmod +x /usr/local/bin/docker-compose
[root@kafka1 ~]# docker-compose version
docker-compose version 1.27.1, build 509cfb99
docker-py version: 4.3.1
CPython version: 3.7.7
OpenSSL version: OpenSSL 1.1.0l  10 Sep 2019

根据环境部署zookeeper，kafka，kafka-manager

Zookeeper集群使用

请确保所布署的 1~3 台服务器网络可以ping通
确保每台主机的2181\2888\3888端口未占用

zookeeper1

准备挂载目录

[root@kafka1 ~]# mkdir -p /root/zk_kafka_cluster/zookeeper1
[root@kafka1 ~]# cd /root/zk_kafka_cluster/zookeeper1
[root@kafka1 zookeeper1]# mkdir {data,datalog}
[root@kafka1 zookeeper1]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  zookeeper1:
    image: zookeeper #镜像
    restart: always # 重启
    container_name: zookeeper1 #容器名
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    volumes:
    - /root/zk_kafka_cluster/zookeeper1/data:/data
    - /root/zk_kafka_cluster/zookeeper1/datalog:/datalog
    environment:
      ZOO_MY_ID: 1 # id
      ZOO_SERVERS: server.1=192.168.78.155:2888:3888;2181 server.2=192.168.78.156:2888:3888;2181 server.3=192.168.78.157:2888:3888;2181
    network_mode: "host"

启动zookeepr1

[root@kafka1 zookeeper1]# docker-compose up -d

查看状态

[root@kafka1 zookeeper1]# docker ps -a
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS               NAMES
61de533366f5        zookeeper           "/docker-entrypoint.…"   11 minutes ago      Up 11 minutes                           zookeeper1
[root@kafka1 zookeeper1]# docker exec -it zookeeper1 /bin/bash
root@kafka1:/apache-zookeeper-3.6.1-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

zookeeper2

准备挂载目录

[root@kafka2 ~]# mkdir -p /root/zk_kafka_cluster/zookeeper2
[root@kafka2 ~]# cd /root/zk_kafka_cluster/zookeeper2
[root@kafka2 zookeeper2]# mkdir {data,datalog}
[root@kafka2 zookeeper2]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  zookeeper2:
    image: zookeeper #镜像
    restart: always # 重启
    container_name: zookeeper2 #容器名
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    volumes:
    - /root/zk_kafka_cluster/zookeeper2/data:/data
    - /root/zk_kafka_cluster/zookeeper2/datalog:/datalog
    environment:
      ZOO_MY_ID: 2 # id
      ZOO_SERVERS: server.1=192.168.78.155:2888:3888;2181 server.2=192.168.78.156:2888:3888;2181 server.3=192.168.78.157:2888:3888;2181
    network_mode: "host"

启动zookeeper2

[root@kafka2 zookeeper2]# docker-compose up -d

查看状态

[root@kafka2 zookeeper2]# docker ps -a
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS               NAMES
316ba6fec210        zookeeper           "/docker-entrypoint.…"   12 minutes ago      Up 12 minutes                           zookeeper2
[root@kafka2 zookeeper2]# docker exec -it zookeeper2 /bin/bash
root@kafka2:/apache-zookeeper-3.6.1-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: leader

zookeeper3

准备挂载目录

[root@kafka3 ~]# mkdir -p /root/zk_kafka_cluster/zookeeper3
[root@kafka3 ~]# cd /root/zk_kafka_cluster/zookeeper3
[root@kafka3 zookeeper3]# mkdir {data,datalog}
[root@kafka3 zookeeper3]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  zookeeper3:
    image: zookeeper #镜像
    restart: always # 重启
    container_name: zookeeper3 #容器名
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    volumes:
    - /root/zk_kafka_cluster/zookeeper3/data:/data
    - /root/zk_kafka_cluster/zookeeper3/datalog:/datalog
    environment:
      ZOO_MY_ID: 3 # id
      ZOO_SERVERS: server.1=192.168.78.155:2888:3888;2181 server.2=192.168.78.156:2888:3888;2181 server.3=192.168.78.157:2888:3888;2181
    network_mode: "host"

启动zookeeper3

[root@kafka3 zookeeper3]# docker-compose up -d

查看状态

[root@kafka3 zookeeper3]# docker ps -a
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS               NAMES
eebed4fe0ccd        zookeeper           "/docker-entrypoint.…"   12 minutes ago      Up 12 minutes                           zookeeper3
[root@kafka3 zookeeper3]# docker exec -it zookeeper3 /bin/bash
root@kafka3:/apache-zookeeper-3.6.1-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

Kafka集群使用

使用前确保各主机可以互相ping通
确保zookeeper的服务列表与各对应的zookeeper的ip与客户端口相同
确保zookeeper集群启动
确保每台主机对应的占用端口号不被占用 kafka1对应9092、 kafka2对应9092、kafka3对应9092、kafka-manager对应9000

分别对每一台kafkax所在的主机修改/etc/hosts，例

192.168.78.155   kafka1
192.168.78.156  kafka2
192.168.78.157  kafka3

启动脚本中没有启动kafka-manager，有需要请自行启动。为了匹配kafka的版本，使用时设置2.1.1即可。

kafka1

准备挂载目录

[root@kafka1 ~]# cd zk_kafka_cluster/
[root@kafka1 zk_kafka_cluster]# mkdir -p kafka1/logs
[root@kafka1 zk_kafka_cluster]# cd kafka1
[root@kafka1 kafka1]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  kafka1:
    image: wurstmeister/kafka # 镜像
    restart: always #重启
    container_name: kafka1 #容器名
    ports:
    - 9092:9092
    - 9999:9999
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.78.155:9092 #绑定发布订阅的端口
      KAFKA_ZOOKEEPER_CONNECT: 192.168.78.155:2181,192.168.78.156:2181,192.168.78.157:2181 #连接zookeeper的服务地址
      KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
      KAFKA_BROKER_ID: 0 #kafka的broker.id，区分不同broker
      KAFKA_LISTENERS: PLAINTEXT://192.168.78.155:9092 #绑定监听9092端口
      JMX_PORT: 9999 # jmx
      KAFKA_MESSAGE_MAX_BYTES: 2000000 #单条消息最大字节数
    volumes:
    - /etc/localtime:/etc/localtime
    - /root/zk_kafka_cluster/kafka1/logs:/kafka
    network_mode: "host"

启动kafka1

[root@kafka1 kafka1]# docker-compose up -d

kafka2

准备挂载目录

[root@kafka2 ~]# cd zk_kafka_cluster/
[root@kafka2 zk_kafka_cluster]# mkdir -p kafka2/logs
[root@kafka2 zk_kafka_cluster]# cd kafka2
[root@kafka2 kafka2]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  kafka2:
    image: wurstmeister/kafka # 镜像
    restart: always #重启
    container_name: kafka2 #容器名
    ports:
    - 9092:9092
    - 9999:9999
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.78.156:9092 #绑定发布订阅的端口
      KAFKA_ZOOKEEPER_CONNECT: 192.168.78.155:2181,192.168.78.156:2181,192.168.78.157:2181 #连接zookeeper的服务地址
      KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
      KAFKA_BROKER_ID: 1 #kafka的broker.id，区分不同broker
      KAFKA_LISTENERS: PLAINTEXT://192.168.78.156:9092 #绑定监听9092端口
      JMX_PORT: 9999 # jmx
      KAFKA_MESSAGE_MAX_BYTES: 2000000 #单条消息最大字节数
    volumes:
    - /etc/localtime:/etc/localtime
    - /root/zk_kafka_cluster/kafka2/logs:/kafka
    network_mode: "host"

启动kafka2

[root@kafka2 kafka2]# docker-compose up -d

kafka3

准备挂载目录

[root@kafka3 ~]# cd zk_kafka_cluster/
[root@kafka3 zk_kafka_cluster]# mkdir -p kafka3/logs
[root@kafka3 zk_kafka_cluster]# cd kafka3
[root@kafka3 kafka3]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  kafka3:
    image: wurstmeister/kafka # 镜像
    restart: always #重启
    container_name: kafka3 #容器名
    ports:
    - 9092:9092
    - 9999:9999
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.78.157:9092 #绑定发布订阅的端口
      KAFKA_ZOOKEEPER_CONNECT: 192.168.78.155:2181,192.168.78.156:2181,192.168.78.157:2181 #连接zookeeper的服务地址
      KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
      KAFKA_BROKER_ID: 2 #kafka的broker.id，区分不同broker
      KAFKA_LISTENERS: PLAINTEXT://192.168.78.157:9092 #绑定监听9092端口
      JMX_PORT: 9999 # jmx
      KAFKA_MESSAGE_MAX_BYTES: 2000000 #单条消息最大字节数
    volumes:
    - /etc/localtime:/etc/localtime
    - /root/zk_kafka_cluster/kafka3/logs:/kafka
    network_mode: "host"

启动kafka3

[root@kafka3 kafka3]# docker-compose up -d

kafka-manager

[root@kafka1 ~]# cd zk_kafka_cluster/
[root@kafka1 zk_kafka_cluster]# mkdir kafka-manager
[root@kafka1 zk_kafka_cluster]# cd kafka-manager/
[root@kafka1 kafka-manager]# vim docker-compose.yml

编写docker-compose.yml文件

version: '3.8'
services:
  kafka-manager:
    image: hlebalbau/kafka-manager:stable
    restart: always
    container_name: kafka-manager
    ports:
    - 9000:9000
    environment:
      ZK_HOSTS: 192.168.68.155:2181,192.168.78.156:2181,192.168.78.157:2181
      KAFKA_BROKERS: 192.168.68.155:9092,192.168.68.156:9092,192.168.68.157:9092
      APPLICATION_SECRET: "random-secret"
      KAFKA_MANAGER_AUTH_ENABLED: "true" # 开启验证
      KAFKA_MANAGER_USERNAME: "admin" # 用户名
      KAFKA_MANAGER_PASSWORD: "admin" # 密码
      KM_ARGS: -Djava.net.preferIPv4Stack=true
      network_mode: "host"

启动kafka-manager

[root@kafka1 kafka-manager]# docker-compose up -d

访问192.168.78.155:9000

账号：admin

密码：admin

手动添加集群即可