1，目的
在了解Docker容器、镜像和仓库基本使用的情况下，可尝试搭建Docker集群，进入Docker云计算时代。
通过本文的学习，可以了解学习Docker集群的搭建和使用，了解学习Docker的基本网络配置。
2，Docker集群模块划分
本文将搭建一个简单的Docker集群，包含三个模块，分别是消息模块、计算模块和测试模块。
消息模块，docker_message
使用redis作为消息中间件，包含三个list：list_message_in、list_message_out和list_message_alive，只部署一份计算模块，docker_worker
启动Python程序，读取docker_message的list_message_in，完成计算后，将结果输出到list_message_out，可以启动多份。另，每秒输出心跳到list_message_alive（空闲时间）测试模块，redis-cli或python程序
通过redis-cli或python程序测试Docker集群是否运行正常
3，消息模块搭建
启动消息模块
# 创建网络，docker有四种网络类型（bridge、container、host和none），我们使用的是none，可以设置静态IP
# docker network create --subnet=172.18.0.0/24 my_network
0b85ab8c7809886bb7fada8c4a8970b5692f11dbd950790f1e9b1e098691f114
# 启动新容器，设置静态IP
# docker run -it --name docker_message -h docker_message -p 26379:6379 --net my_network --ip 172.18.0.2 --add-host docker_message:172.18.0.2 learn/redis:v2 /bin/sh /etc/rc.local
参数说明
- -it，交互方式启动
- --name docker_message，指定新容器的名称是docker_message
- -h docker_message，指定新容器的主机名是docker_message
- -p 26379:6379，指定宿主机与docker容器的端口映射，宿主机的26379对应docker容器的6379
- --net my_network，指定网络是新创建的网络my_network，IP段是172.18.0.0/24
- --ip 172.18.0.2，指定容器的IP为静态IP 172.18.0.2
- --add-host docker_message:172.18.0.2，增加容器的host配置，docker_message:172.18.0.2
- learn/redis:v2，指定启动容器对应的镜像是learn/redis:v2，可以是镜像ID
- /bin/sh /etc/rc.local，指定容器启动后，执行shell脚本是/etc/rc.local
进入到容器，安装redis，并启动测试
root@docker_message:/# apt-get install redis-server
root@docker_message:/# service redis-server start
Starting redis-server: redis-server.
root@docker_message:/# redis-cli -h docker_message
docker_message:6379> set a 00
OK
docker_message:6379> get a
"00"
测试Docker容器的网络设置
- 通过宿主机和外网机器（非172.18.0.0/24网段）的机器，连接docker_message的redis服务，测试是否正常
# 宿主机的测试
kevin@apple:~$ redis-cli -h 172.18.0.2 -p 6379
172.17.0.2:6379> get a
(nil)
172.17.0.2:6379> keys *
(empty list or set)
172.17.0.2:6379> set a 11
OK
172.17.0.2:6379> get a
"11"
# 外网机器的测试
kevin@orange:~/docker$ redis-cli -h 192.168.1.89 -p 26379
192.168.1.89:26379> set a 22
OK
192.168.1.89:26379> get a
"22"
- 在docker_message查看网络连接情况，发现6379端口有来自192.168.1.32（外网机器）和172.18.0.1（宿主机）的TCP连接，说明网络设置OK
root@docker_message:/# netstat -pan
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:6379 0.0.0.0:* LISTEN -
tcp 0 0 127.0.0.11:33007 0.0.0.0:* LISTEN -
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 30/sshd
tcp 0 0 172.18.0.2:6379 172.18.0.1:52440 ESTABLISHED -
tcp 0 0 172.18.0.2:6379 192.168.1.32:36922 ESTABLISHED -
tcp6 0 0 :::6379 :::* LISTEN -
tcp6 0 0 :::22 :::* LISTEN 30/sshd
udp 0 0 127.0.0.11:60720 0.0.0.0:* -
Active UNIX domain sockets (servers and established)
Proto RefCnt Flags Type State I-Node PID/Program name Path
修改docker容器的开机脚本
# vi /etc/rc.local
####!/bin/sh -e
#
# rc.local
#
# This script is executed at the end of each multiuser runlevel.
# Make sure that the script will "exit 0" on success or any other
# value on error.
#
# In order to enable or disable this script just change the execution
# bits.
#
# By default this script does nothing.
service redis-server start
/bin/bash
exit 0
4，计算模块搭建
启动一个docker容器
# docker run -it --name docker_worker1 -h docker_worker1 --net my_network --add-host docker_message:172.18.0.2 learn/redis:v2 /bin/bash
参数说明
- --net my_network，指定容器使用的网络是my_network，这样可以与docker_message内网通信
- --add-host docker_message:172.18.0.2，增加容器的host配置，docker_message:172.18.0.2，可以通过docker_message连接消息模块
- 备注：可以通过-e docker_message=172.18.0.2增加环境变量
网络测试
root@docker_worker1:/# ifconfig
eth0 Link encap:Ethernet HWaddr 02:42:ac:12:00:03
inet addr:172.18.0.3 Bcast:0.0.0.0 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:12 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:1856 (1.8 KB) TX bytes:0 (0.0 B)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
root@docker_worker1:/# redis-cli -h docker_message -p 6379
docker_message:6379> set a 22
OK
docker_message:6379> get a
"22"
编写Python脚本
- 安装python包
# apt-get install python-redis
- 创建python脚本
root@docker_worker1:/home/visual/python# vi /home/visual/python/worker.py
#coding:utf8
import redis, socket, time
# 计算函数
def worker():
# 连接redis
# docker_message = "192.168.1.89"
docker_message = "docker_message"
r = redis.Redis(host=docker_message, port=6379, db=0)
# 从list_message_in获取数据，计算后写入到list_message_out
list_message_in = "list_message_in"
list_message_out = "list_message_out"
list_message_alive = "list_message_alive"
hostname = socket.gethostname()
while(True):
str_message = r.rpop(list_message_in)
# 如果获取的消息为空，则休息1秒再看是否有数据
if str_message == None:
if r.llen(list_message_alive) >= 100:
r.delete(list_message_alive)
str_alive_message = hostname + " is alive at " + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
r.lpush(list_message_alive, str_alive_message)
time.sleep(1)
continue
# 完成表达式的计算，并返回结果
try:
str_message = str_message + "=" + str(eval(str_message))
except Exception,e:
print str_message,e
str_response = "worker %s return %s" % ( hostname, str_message )
r.lpush(list_message_out, str_response)
# print str_response
if __name__ == "__main__":
worker()
- 容器中启动Python脚本
root@docker_worker1:/home/visual/python# python worker.py 2>&1 &
- 测试Python运行正常
root@apple:/home/kevin/docker/docker_cluster# redis-cli -p 26379
127.0.0.1:26379> keys *
1) "list_message_out"
2) "list_message_alive"
127.0.0.1:26379> llen list_message_alive
(integer) 67
127.0.0.1:26379> llen list_message_alive
(integer) 68
127.0.0.1:26379> llen list_message_alive
(integer) 69
127.0.0.1:26379> llen list_message_alive
(integer) 70
127.0.0.1:26379> lpush list_message_in "1+2-3+3*3"
(integer) 1
127.0.0.1:26379> lrange list_message_out 0 3
1) "worker docker_worker1 return 1+2-3+3*3=9"
2) "1+2+3*2=9"
3) "1+2+3*2"
4) "None"
添加开机启动
root@docker_worker1:/# cat /etc/rc.local
#!/bin/sh -e
#
# rc.local
#
# This script is executed at the end of each multiuser runlevel.
# Make sure that the script will "exit 0" on success or any other
# value on error.
#
# In order to enable or disable this script just change the execution
# bits.
#
# By default this script does nothing.
# service ssh start
# service redis-server start
python /home/visual/python/worker.py 2>&1 &
/bin/bash
exit 0
重新打包生成镜像learn/worker:v1，可还原部署多份容器
kevin@apple:~$ docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
a5e5efb3f995 learn/redis:v2 "/bin/bash" 2 hours ago Exited (0) 14 seconds ago docker_worker1
ad7291dea597 learn/redis:v2 "/bin/sh /etc/rc.l..." 2 hours ago Up 2 hours 0.0.0.0:26379->6379/tcp docker_message
cbbbe7a5d47a learn/nginx:v2 "/bin/sh /etc/rc.l..." 6 hours ago Exited (255) 3 hours ago 0.0.0.0:8000->80/tcp, 0.0.0.0:8001->81/tcp nginx_test
93a1b9d39683 ubuntu "bash" 2 days ago Exited (0) 7 hours ago zealous_noether
abdc084f9821 hello-world "/hello" 2 days ago Exited (0) 26 hours ago sleepy_clarke
kevin@apple:~$ docker commit a5e5efb3f995 learn/worker:v1
sha256:abb80d7a396cca5d99d3253a85a9629413a962e1dd7a18c77b53ce97d1cbd0f9
kevin@apple:~$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
learn/worker v1 abb80d7a396c 16 seconds ago 325MB
learn/redis v2 9385e5566f04 2 hours ago 324MB
learn/nginx v2 3393d626158d 6 hours ago 370MB
learn/nginx v1 ab92edd21696 8 hours ago 370MB
learn/redis v1 619ec66bf6dd 18 hours ago 324MB
learn/visual_init v1 56a4eab7dc5b 44 hours ago 321MB
registry.cn-beijing.aliyuncs.com/zhangsp/aiwen visual_init 56a4eab7dc5b 44 hours ago 321MB
ubuntu latest 14f60031763d 5 days ago 120MB
hello-world latest 1815c82652c0 5 weeks ago 1.84kB
启动两份docker_worker
# docker run -it --name docker_worker1 -h docker_worker1 --net my_network --add-host docker_message:172.18.0.2 learn/worker:v1 /bin/sh /etc/rc.local
# docker run -it --name docker_worker2 -h docker_worker2 --net my_network --add-host docker_message:172.18.0.2 learn/worker:v1 /bin/sh /etc/rc.local
5，执行测试模块
截止目前，Docker简单集群已经搭建OK。我们共有三个Docker容器，分别是docker_message、docker_worker1、docker_worker2，如下：
root@apple:/home/kevin/docker/docker_cluster# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b454ac04f6a0 learn/worker:v1 "/bin/sh /etc/rc.l..." About an hour ago Up About an hour 6379/tcp docker_worker2
ecdb7a63da09 learn/worker:v1 "/bin/sh /etc/rc.l..." About an hour ago Up About an hour 6379/tcp docker_worker1
ad7291dea597 learn/redis:v2 "/bin/sh /etc/rc.l..." 3 hours ago Up 3 hours 0.0.0.0:26379->6379/tcp docker_message
cbbbe7a5d47a learn/nginx:v2 "/bin/sh /etc/rc.l..." 7 hours ago Exited (255) 4 hours ago 0.0.0.0:8000->80/tcp, 0.0.0.0:8001->81/tcp nginx_test
93a1b9d39683 ubuntu "bash" 2 days ago Exited (0) 8 hours ago zealous_noether
abdc084f9821 hello-world "/hello" 2 days ago Exited (0) 27 hours ago sleepy_clarke
测试一下Docker集群服务是否正常，可在另外一台机器上，通过redis-cli测试Docker集群（3个docker容器）。当然可以通过python或其他程序连接redis测试。
kevin@orange:~/docker$ redis-cli -h 192.168.1.89 -p 26379
192.168.1.89:26379> keys *
1) "list_message_out"
2) "list_message_alive"
192.168.1.89:26379> lrange list_message_alive 0 3
1) "docker_worker1 is alive at 2017-07-26 08:34:25"
2) "docker_worker2 is alive at 2017-07-26 08:34:24"
3) "docker_worker1 is alive at 2017-07-26 08:34:24"
4) "docker_worker2 is alive at 2017-07-26 08:34:23"
192.168.1.89:26379> lpush list_message_in "1+2+3"
(integer) 1
192.168.1.89:26379> lrange list_message_out 0 3
1) "worker docker_worker1 return 1+2+3=6"
2) "worker docker_worker1 return 1+2-3+3*3=9"
3) "1+2+3*2=9"
4) "1+2+3*2"
192.168.1.89:26379> lpush list_message_in "1+2+3"
(integer) 1
192.168.1.89:26379> lpush list_message_in "1+2+3"
(integer) 1
192.168.1.89:26379> lpush list_message_in "1+2+3"
(integer) 1
192.168.1.89:26379> lpush list_message_in "1+2+3"
(integer) 1
192.168.1.89:26379> lrange list_message_out 0 3
1) "worker docker_worker1 return 1+2+3=6"
2) "worker docker_worker2 return 1+2+3=6"
3) "worker docker_worker1 return 1+2+3=6"
4) "worker docker_worker2 return 1+2+3=6"
测试的Python代码如下：
#coding:utf8
import redis, time
def test():
# 连接redis
docker_message = "192.168.1.89"
# docker_message = "docker_message"
r = redis.Redis(host=docker_message, port=26379, db=0)
# list_message_in是docker集群的消息输入列表
# list_message_out是docker集群的消息输出列表
# list_message_alive是docker_worker的心跳存活数据列表
list_message_in = "list_message_in"
list_message_out = "list_message_out"
list_message_alive = "list_message_alive"
while(True):
# r.lpush(list_message_in, "1+2+2+10/2")
str_response = r.brpop([list_message_out,list_message_alive])
print time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time()) ), str_response
time.sleep(0.2)
if __name__ == "__main__":
test()
通过上述测试，可以发现：
- 能够正常连接到docker_message模块，通过192.168.1.89:26379，或者172.18.0.1:26378，或者172.18.0.2:6378均可以正常连接。
- 通过list_message_alive列表，可以发现docker_worker1和docker_worker2均正常存活，有心跳报告，并且list_message_alive的列表长度不超过100。
- 通过向list_message_in列表增加数据，可以看到list_message_out有数据返回，说明集群运行正常。
- 通过list_message_out的返回数据，可以看到来自不同的worker，包含docker_worker1和docker_worker2，说明简单的docker集群已经OK。
6，一些坑
如何run一个镜像的时候，可以在容器中启动多个进程
首先，由容器创建镜像之前，最好修改容器中的/etc/rc.local文件，在exit之前执行/bin/bash，在/bin/bash之前可以启动自己的服务，例如redis或Python脚本。然后，执行run镜像的时候，增加/bin/sh /etc/rc.local即可。容器和镜像的规范化管理
首先，镜像的命名可以采用learn/redis:v1的方式来命名，learn是项目名称，redis是模块名称，v1是版本号。然后，容器的命名可以采用模块名称进行命名。最后，不用的镜像和容器，尽可能删除掉，避免越多越乱。
