RabbitMQ集群部署文档

拓扑结构介绍：

192.168.43.125 rabbitMQ01

192.168.43.126 rabbitMQ02

192.168.43.127 rabbitMQ03

rabbitMQ版本：rabbitmq-server-3.7.4-1.el7.noarch.rpm

erlang版本：erlang-20.2.2-1.el7.centos.x86_64.rpm

OS ：CentOS Linux release 7.7.1908 (Core)

一、系统基本配置（所有节点）

1.关闭SELINUX及防火墙（略）

2.配置ntp服务

[root@localhost opt]# yum install ntp -y

[root@localhost opt]# ntpdate ntp.aliyun.com

30 Aug 10:02:26 ntpdate[1737]: step time server 203.107.6.88 offset -28799.327218 sec

[root@localhost opt]#

[root@localhost opt]# grep '^server' /etc/ntp.conf

server ntp.aliyun.com

[root@localhost opt]# systemctl enable ntpd

Created symlink from /etc/systemd/system/multi-user.target.wants/ntpd.service to /usr/lib/systemd/system/ntpd.service.

[root@localhost opt]# systemctl start ntpd

[root@localhost opt]# date

Sun Aug 30 10:05:01 CST 2020

3.修改/etc/hosts

[root@localhost opt]# cat /etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4

::1 localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.43.125 rabbitMQ01

192.168.43.126 rabbitMQ02

192.168.43.127 rabbitMQ03

[root@localhost opt]#

二、安装rabbitMQ（所有节点）

参阅：《RabbitMQ安装配置文档》

三、配置集群

1.同步cookie文件

rabbitMQ01上：

[root@rabbitMQ01 opt]# scp /var/lib/rabbitmq/.erlang.cookie rabbitMQ02:/var/lib/rabbitmq/.erlang.cookie

root@rabbitmq02's password:

.erlang.cookie 100% 20 8.5KB/s 00:00

[root@rabbitMQ01 opt]# scp /var/lib/rabbitmq/.erlang.cookie rabbitMQ03:/var/lib/rabbitmq/.erlang.cookie

root@rabbitmq03's password:

.erlang.cookie 100% 20 26.3KB/s 00:00

[root@rabbitMQ01 opt]# cat /var/lib/rabbitmq/.erlang.cookie

SWRTHKCJDJEFXIYOJSHX

[root@rabbitMQ01 opt]# ll /var/lib/rabbitmq/.erlang.cookie

-r-------- 1 rabbitmq rabbitmq 20 Aug 29 00:00 /var/lib/rabbitmq/.erlang.cookie

[root@rabbitMQ01 opt]#

rabbitMQ02上：

[root@rabbitMQ02 ~]# cat /var/lib/rabbitmq/.erlang.cookie

SWRTHKCJDJEFXIYOJSHX

[root@rabbitMQ02 ~]# ll /var/lib/rabbitmq/.erlang.cookie

-r-------- 1 rabbitmq rabbitmq 20 Aug 30 10:37 /var/lib/rabbitmq/.erlang.cookie

[root@rabbitMQ02 ~]#

rabbitMQ03上：

[root@rabbitMQ03 ~]# cat /var/lib/rabbitmq/.erlang.cookie

SWRTHKCJDJEFXIYOJSHX

[root@rabbitMQ03 ~]# ll /var/lib/rabbitmq/.erlang.cookie

-r-------- 1 rabbitmq rabbitmq 20 Aug 30 10:37 /var/lib/rabbitmq/.erlang.cookie

[root@rabbitMQ03 ~]#

2.加入集群（rabbitMQ02及rabbitMQ03上操作）

rabbitMQ02上：

[root@rabbitMQ02 home]# systemctl start rabbitmq-server

[root@rabbitMQ02 home]# ps -ef | grep rabbitmq

rabbitmq 3335 1 0 10:41 ? 00:00:00 /usr/lib64/erlang/erts-9.2/bin/epmd -daemon

rabbitmq 6038 1 43 11:19 ? 00:00:06 /usr/lib64/erlang/erts-9.2/bin/beam.smp -W w -A 64 -P 1048576 -t 5000000 -stbt db -zdbbl 1280000 -K true -- -root /usr/lib64/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.7.4/ebin -noshell -noinput -s rabbit boot -sname rabbit@rabbitMQ02 -boot start_sasl -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit lager_log_root "/var/log/rabbitmq" -rabbit lager_default_file "/var/log/rabbitmq/rabbit@rabbitMQ02.log" -rabbit lager_upgrade_file "/var/log/rabbitmq/rabbit@rabbitMQ02_upgrade.log" -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/plugins:/usr/lib/rabbitmq/lib/rabbitmq_server-3.7.4/plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@rabbitMQ02-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@rabbitMQ02" -kernel inet_dist_listen_min 25672 -kernel inet_dist_listen_max 25672

rabbitmq 6359 6038 0 11:19 ? 00:00:00 erl_child_setup 1024

rabbitmq 6383 6359 0 11:19 ? 00:00:00 inet_gethost 4

rabbitmq 6384 6383 0 11:19 ? 00:00:00 inet_gethost 4

root 6392 1556 0 11:19 pts/0 00:00:00 grep --color=auto rabbitmq

[root@rabbitMQ02 home]# rabbitmqctl stop_app

Stopping rabbit application on node rabbit@rabbitMQ02 ...

[root@rabbitMQ02 home]# ps -ef | grep rabbitmq

rabbitmq 3335 1 0 10:41 ? 00:00:00 /usr/lib64/erlang/erts-9.2/bin/epmd -daemon

rabbitmq 6038 1 21 11:19 ? 00:00:06 /usr/lib64/erlang/erts-9.2/bin/beam.smp -W w -A 64 -P 1048576 -t 5000000 -stbt db -zdbbl 1280000 -K true -- -root /usr/lib64/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.7.4/ebin -noshell -noinput -s rabbit boot -sname rabbit@rabbitMQ02 -boot start_sasl -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit lager_log_root "/var/log/rabbitmq" -rabbit lager_default_file "/var/log/rabbitmq/rabbit@rabbitMQ02.log" -rabbit lager_upgrade_file "/var/log/rabbitmq/rabbit@rabbitMQ02_upgrade.log" -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/plugins:/usr/lib/rabbitmq/lib/rabbitmq_server-3.7.4/plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@rabbitMQ02-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@rabbitMQ02" -kernel inet_dist_listen_min 25672 -kernel inet_dist_listen_max 25672

rabbitmq 6359 6038 0 11:19 ? 00:00:00 erl_child_setup 1024

rabbitmq 6383 6359 0 11:19 ? 00:00:00 inet_gethost 4

rabbitmq 6384 6383 0 11:19 ? 00:00:00 inet_gethost 4

root 6528 1556 0 11:20 pts/0 00:00:00 grep --color=auto rabbitmq

[root@rabbitMQ02 home]# rabbitmqctl join_cluster rabbit@rabbitMQ01 # 这里是磁盘节点，内存节点用 rabbitmqctl join_cluster --ram rabbit@rabbitMQ01

Clustering node rabbit@rabbitMQ02 with rabbit@rabbitMQ01

[root@rabbitMQ02 home]#

rabbitMQ03上：

[root@rabbitMQ03 rabbitmq]# systemctl start rabbitmq-server

[root@rabbitMQ03 rabbitmq]# rabbitmqctl stop_app

Stopping rabbit application on node rabbit@rabbitMQ03 ...

[root@rabbitMQ03 rabbitmq]# rabbitmqctl join_cluster rabbit@rabbitMQ01 # 这里是磁盘节点，内存节点用 rabbitmqctl join_cluster --ram rabbit@rabbitMQ01

Clustering node rabbit@rabbitMQ03 with rabbit@rabbitMQ01

[root@rabbitMQ03 rabbitmq]#

3.验证集群状态

rabbitMQ01上：

[root@rabbitMQ01 rabbitmq]# rabbitmqctl cluster_status

Cluster status of node rabbit@rabbitMQ01 ...

[{nodes,[{disc,[rabbit@rabbitMQ01,rabbit@rabbitMQ02,rabbit@rabbitMQ03]}]},

{running_nodes,[rabbit@rabbitMQ01]},

{cluster_name,<<"rabbit@rabbitMQ01">>},

{partitions,[]},

{alarms,[{rabbit@rabbitMQ01,[]}]}]

[root@rabbitMQ01 rabbitmq]#

rabbitMQ02上：

[root@rabbitMQ02 home]# rabbitmqctl start_app

Starting node rabbit@rabbitMQ02 ...

completed with 3 plugins.

[root@rabbitMQ02 home]# rabbitmqctl cluster_status

Cluster status of node rabbit@rabbitMQ02 ...

[{nodes,[{disc,[rabbit@rabbitMQ01,rabbit@rabbitMQ02,rabbit@rabbitMQ03]}]},

{running_nodes,[rabbit@rabbitMQ03,rabbit@rabbitMQ01,rabbit@rabbitMQ02]},

{cluster_name,<<"rabbit@rabbitMQ01">>},

{partitions,[]},

{alarms,[{rabbit@rabbitMQ03,[]},

{rabbit@rabbitMQ01,[]},

{rabbit@rabbitMQ02,[]}]}]

[root@rabbitMQ02 home]#

rabbitMQ03上：

[root@rabbitMQ03 rabbitmq]# rabbitmqctl start_app

Starting node rabbit@rabbitMQ03 ...

completed with 3 plugins.

[root@rabbitMQ03 rabbitmq]# rabbitmqctl cluster_status

Cluster status of node rabbit@rabbitMQ03 ...

[{nodes,[{disc,[rabbit@rabbitMQ01,rabbit@rabbitMQ02,rabbit@rabbitMQ03]}]},

{running_nodes,[rabbit@rabbitMQ01,rabbit@rabbitMQ02,rabbit@rabbitMQ03]},

{cluster_name,<<"rabbit@rabbitMQ01">>},

{partitions,[]},

{alarms,[{rabbit@rabbitMQ01,[]},

{rabbit@rabbitMQ02,[]},

{rabbit@rabbitMQ03,[]}]}]

[root@rabbitMQ03 rabbitmq]#

4.创建用户（任意节点执行）

这里：我们选用rabbitMQ02节点进行操作

[root@rabbitMQ02 home]# rabbitmqctl add_user testcluster testcluster # 添加用户

Adding user "testcluster" ...

[root@rabbitMQ02 home]# rabbitmqctl set_user_tags testcluster administrator # 加入管理员

Setting tags for user "testcluster" to [administrator] ...

[root@rabbitMQ02 home]# rabbitmqctl set_permissions -p / testcluster ".*" ".*" ".*" # 对根目录设置消费权限

Setting permissions for user "testcluster" in vhost "/" ...

[root@rabbitMQ02 home]#

5.可以使用该账号登录rabbitMQ03进行测试： http://192.168.43.127:15672/#/

Protocol Node Bound to Port

-----------------------------------------------------

amqp rabbit@rabbitMQ01 :: 5672

amqp rabbit@rabbitMQ02 :: 5672

amqp rabbit@rabbitMQ03 :: 5672

clustering rabbit@rabbitMQ01 :: 25672

clustering rabbit@rabbitMQ02 :: 25672

clustering rabbit@rabbitMQ03 :: 25672

http rabbit@rabbitMQ01 :: 15672

http rabbit@rabbitMQ02 :: 15672

http rabbit@rabbitMQ03 :: 15672

6.高可用集群策略的配置（在任意一个节点上执行）

说明一下：RabbitMQ 有三种模式：单机模式、普通集群模式、镜像集群模式。

以上配置完毕后，模式为普通集群模式，集群数据不共享，如果要做到高可用，需要采用镜像集群模式，

我们这里生成一条策略以实现镜像集群模式。

将所有队列设置为镜像队列，即队列会被复制到各个节点，各个节点状态保持一致：

[root@rabbitMQ02 home]# rabbitmqctl set_policy ha-all "^" '{"ha-mode":"all"}'

Setting policy "ha-all" for pattern "^" to "{"ha-mode":"all"}" with priority "0" for vhost "/" ...

四、配置HAProxy实现负载均衡的调度

由于测试环境，我们任意找一台服务器安装HAproxy即可，生产环境尽量单独使用1台服务器充当调度器角色。

我们选用rabbitMQ03来进行测试：

1.安装HAProxy

[root@rabbitMQ03 rabbitmq]# yum install haproxy -y

Loaded plugins: fastestmirror

Loading mirror speeds from cached hostfile

* base: mirrors.aliyun.com

* extras: mirrors.aliyun.com

* updates: mirrors.aliyun.com

Package haproxy-1.5.18-9.el7.x86_64 already installed and latest version

Nothing to do

2.调整haproxy配置文件

[root@rabbitMQ03 rabbitmq]# grep -v "^#" /etc/haproxy/haproxy.cfg | grep -v "^$"

global

# to have these messages end up in /var/log/haproxy.log you will

# need to:

# 1) configure syslog to accept network log events. This is done

# by adding the '-r' option to the SYSLOGD_OPTIONS in

# /etc/sysconfig/syslog

# 2) configure local2 events to go to the /var/log/haproxy.log

# file. A line like the following can be added to

# /etc/sysconfig/syslog

# local2.* /var/log/haproxy.log

log 127.0.0.1 local2

chroot /var/lib/haproxy

pidfile /var/run/haproxy.pid

maxconn 4000

user haproxy

group haproxy

daemon

# turn on stats unix socket

stats socket /var/lib/haproxy/stats

defaults

mode http

log global

option httplog

option dontlognull

option http-server-close

option forwardfor except 127.0.0.0/8

option redispatch

retries 3

timeout http-request 10s

timeout queue 1m

timeout connect 10s

timeout client 1m

timeout server 1m

timeout http-keep-alive 10s

timeout check 10s

maxconn 3000

listen admin_stats

bind *:9188

mode http

log 127.0.0.1 local3 err

stats refresh 60s

stats uri /haproxy_status

stats realm welcome login\ Haproxy

stats auth admin:123.com

stats hide-version

stats admin if TRUE

listen rabbitmq_cluster

bind *:5673

mode tcp

balance roundrobin

server rabbitMQ01 192.168.43.125:5672 check inter 2000 rise 2 fall 3 weight 1

server rabbitMQ02 192.168.43.126:5672 check inter 2000 rise 2 fall 3 weight 1

server rabbitMQ03 192.168.43.127:5672 check inter 2000 rise 2 fall 3 weight 1

3.启动HAProxy

[root@rabbitMQ03 rabbitmq]# systemctl enable haproxy

[root@rabbitMQ03 rabbitmq]# systemctl restart haproxy

[root@rabbitMQ03 rabbitmq]# systemctl status haproxy

● haproxy.service - HAProxy Load Balancer

Loaded: loaded (/usr/lib/systemd/system/haproxy.service; disabled; vendor preset: disabled)

Active: active (running) since Sun 2020-08-30 13:09:06 CST; 7s ago

Main PID: 16984 (haproxy-systemd)

CGroup: /system.slice/haproxy.service

├─16984 /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid

├─16985 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds

└─16986 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds

Aug 30 13:09:06 rabbitMQ03 systemd[1]: Started HAProxy Load Balancer.

Aug 30 13:09:06 rabbitMQ03 haproxy-systemd-wrapper[16984]: haproxy-systemd-wrapper: executing /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds

Aug 30 13:09:06 rabbitMQ03 haproxy-systemd-wrapper[16984]: [WARNING] 242/130906 (16985) : parsing [/etc/haproxy/haproxy.cfg:45] : 'option httplog' not usable with proxy 'rab...n tcplog'.

Aug 30 13:09:06 rabbitMQ03 haproxy-systemd-wrapper[16984]: [WARNING] 242/130906 (16985) : config : 'option forwardfor' ignored for proxy 'rabbitmq_cluster' as it requires HTTP mode.

Hint: Some lines were ellipsized, use -l to show in full.

[root@rabbitMQ03 rabbitmq]#

4.访问测试

http://192.168.43.127:9188/haproxy_status

五、利用Python通过页面观察rabbitMQ集群的状态变化

注意：这里连接的IP和端口是HAProxy的调度IP与端口（192.168.43.127:5673）。

procedure.py # 生产者

import pika

credentials = pika.PlainCredentials('testcluster', 'testcluster')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.43.127', 5673, '/', credentials))

channel = connection.channel()

channel.queue_declare(queue='hello')

channel.basic_publish(exchange='',

routing_key='hello',

body='Hello5 World!')

print(" [x] Sent 'Hello World!'")

connection.close()

********************************************************************************************************

consumer.py # 消费者

import pika

def callback(ch, method, properties, body):

print(" [x] Received %r" % body)

credentials = pika.PlainCredentials('testcluster', 'testcluster')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.43.127', 5673, '/', credentials))

channel = connection.channel()

channel.queue_declare(queue='hello')

channel.basic_consume(queue='hello', on_message_callback=callback, auto_ack=True)

print(' [*] Waiting for messages. To exit press CTRL+C')

channel.start_consuming()

connection.close()

分别访问下列连接（testcluster/testcluster）来不断测试镜像模式与普通模式的特性。

http://192.168.43.125:15672/#/

http://192.168.43.126:15672/#/

http://192.168.43.127:15672/#/

参阅：

https://www.rabbitmq.com/ha.html

https://nieoding-dis-doc.readthedocs.io/zh/latest/rabbitmq/

https://www.cnblogs.com/mrChangChang/p/11889203.html

https://blog.csdn.net/zhuyu19911016520/article/details/80206202