安装
yum install erlang-R16B-03.18.el7
wget http://www.rabbitmq.com/releases/rabbitmq-server/v3.6.8/rabbitmq-server-3.6.8-1.el7.noarch.rpm
rpm --import https://www.rabbitmq.com/rabbitmq-release-signing-key.asc
yum install rabbitmq-server-3.6.8-1.el7.noarch.rpm
将sns-mq01 的 /var/lib/rabbitmq/.erlang.cookie 拷贝到 sns-mq02 和 sns-mq03上
chown rabbitmq:rabbitmq .erlang.cookie
chmod 400 .erlang.cookie
vim /etc/rabbitmq/rabbitmq.config
[{rabbit,[{vm_memory_high_watermark,0.6}, {cluster_partition_handling, pause_minority}]}].
vim /etc/rabbitmq/rabbitmq-env.conf
RABBITMQ_MNESIA_BASE=/data/rabbitmq/mnesia
RABBITMQ_LOG_BASE=/data/rabbitmq/log
logrotate vim /etc/logrotate.d/rabbitmq-server
/data/rabbitmq/log/*.log {
rotate 7
daily
missingok
dateext
dateformat .%Y%m%d
notifempty
sharedscripts
postrotate
/usr/sbin/rabbitmqctl rotate_logs > /dev/null
endscript
}
mkdir /data/rabbitmq
mkdir /data/rabbitmq/mnesia
mkdir /data/rabbitmq/log
chown rabbitmq:rabbitmq /data/rabbitmq/mnesia/
chown rabbitmq:rabbitmq /data/rabbitmq/log/
chmod 750 /data/rabbitmq/mnesia/
chmod 750 /data/rabbitmq/log/
rabbitmq-plugins enable rabbitmq_management 开启管理界面
rabbitmq-plugins enable rabbitmq_top 开启内存占用监控
systemctl start rabbitmq-server.service
rabbitmqctl add_vhost vhost01
rabbitmqctl add_user red xiaohongshu
rabbitmqctl set_permissions -p vhost01 red "." "." ".*"
sns-mq02
systemctl stop rabbitmq-server.service
systemctl start rabbitmq-server.service
rabbitmqctl stop_app
rabbitmqctl reset
rabbitmqctl join_cluster rabbit@sns-mq01
rabbitmqctl start_app
sns-mq03
systemctl stop rabbitmq-server.service
systemctl start rabbitmq-server.service
rabbitmqctl stop_app
rabbitmqctl reset
rabbitmqctl join_cluster --ram rabbit@sns-mq01
rabbitmqctl start_app
LoadBanlance
sns-lb-rabbitmq.devops.xiaohongshu.com: 5670
sns-asura-staging02
runit
vim /etc/service/mq.test/run
#!/bin/bash
exec 2>&1
exec chpst -u deploy -e /data/sns.asura/.env /data/.virtualenvs/sns.asura/bin/python /data/sns.asura/asura/service/mq.py
Rabbitmq Management Plugin
rabbitmq-plugins enable rabbitmq_management
rabbitmqctl set_user_tags red administrator
修改mq03的mnesia位置
rabbitmqctl stop_app
在mq02上: rabbitmqctl forget_cluster_node rabbit@sns-mq03
vim /etc/rabbitmq/rabbitmq-env.conf
RABBITMQ_MNESIA_BASE=/data/rabbitmq/mnesia
RABBITMQ_LOG_BASE=/data/rabbitmq/log
mkdir /data/rabbitmq/mnesia
mkdir /data/rabbitmq/log
chown rabbitmq:rabbitmq /data/rabbitmq/mnesia/
chown rabbitmq:rabbitmq /data/rabbitmq/log/
chmod 750 /data/rabbitmq/mnesia/
chmod 750 /data/rabbitmq/log/
systemctl stop rabbitmq-server.service
systemctl start rabbitmq-server.service
rabbitmqctl stop_app
rabbitmqctl reset
rabbitmqctl join_cluster --ram rabbit@sns-mq01
rabbitmqctl start_app
修改mq02的mnesia位置
rabbitmqctl stop_app
在mq03上: rabbitmqctl forget_cluster_node rabbit@sns-mq02
vim /etc/rabbitmq/rabbitmq-env.conf
RABBITMQ_MNESIA_BASE=/data/rabbitmq/mnesia
RABBITMQ_LOG_BASE=/data/rabbitmq/log
mkdir /data/rabbitmq/mnesia
mkdir /data/rabbitmq/log
chown rabbitmq:rabbitmq /data/rabbitmq/mnesia/
chown rabbitmq:rabbitmq /data/rabbitmq/log/
chmod 750 /data/rabbitmq/mnesia/
chmod 750 /data/rabbitmq/log/
systemctl stop rabbitmq-server.service
systemctl start rabbitmq-server.service
rabbitmqctl stop_app
rabbitmqctl reset
rabbitmqctl join_cluster rabbit@sns-mq01
rabbitmqctl start_app
修改mq01的mnesia位置
rabbitmqctl stop_app
在mq03上: rabbitmqctl forget_cluster_node rabbit@sns-mq01
vim /etc/rabbitmq/rabbitmq-env.conf
RABBITMQ_MNESIA_BASE=/data/rabbitmq/mnesia
RABBITMQ_LOG_BASE=/data/rabbitmq/log
mkdir /data/rabbitmq/
mkdir /data/rabbitmq/mnesia
mkdir /data/rabbitmq/log
chown rabbitmq:rabbitmq /data/rabbitmq/mnesia/
chown rabbitmq:rabbitmq /data/rabbitmq/log/
chmod 750 /data/rabbitmq/mnesia/
chmod 750 /data/rabbitmq/log/
systemctl stop rabbitmq-server.service
systemctl start rabbitmq-server.service
rabbitmqctl stop_app
rabbitmqctl reset
rabbitmqctl join_cluster rabbit@sns-mq02
rabbitmqctl start_app
设置mq03为内存节点
rabbitmqctl stop_app
rabbitmqctl change_cluster_node_type ram
rabbitmqctl start_app
设置内存
rabbitmqctl set_vm_memory_high_watermark 0.6
坑
坑到死，上面搭的老版本经常出一些bug，2个月一个版本，一堆bugfix我也是服气的
还不能平滑升级，经常有一些queue莫名其妙crash, 节点间负载不均。
清理 crashed queue
import os
cmd = """rabbitmqctl eval '{ok, Q} = rabbit_amqqueue:lookup(rabbit_misc:r(<<"vhost_name">>, queue, <<"queue_name">>)), rabbit_amqqueue:delete_crashed(Q).'"""
def restart_queue(vhost):
print "in**********"
queues = os.popen('rabbitmqctl list_queues -p ' + vhost).read()
print queues
dir(queues)
queues = queues.split('\n')[1:]
result = []
for queue in queues:
if queue:
queue = queue.split()[0]
result.append(queue)
command = cmd.replace('vhost_name', vhost)
command = command.replace('queue_name', queue)
print command
os.system(command)
restart_queue('vhost01')
restart_queue('vhost_celery_online')
restart_queue('vhost_celery_staging')
import os
import requests
import json
from requests.auth import HTTPBasicAuth
vhost = "vhost_celery_online"
cmd = """rabbitmqctl eval '{ok, Q} = rabbit_amqqueue:lookup(rabbit_misc:r(<<"vhost_name">>, queue, <<"queue_name">>)), rabbit_amqqueue:delete_crashed(Q).'"""
cmd = cmd.replace('vhost_name', vhost);
queues = requests.get('http://127.0.0.1:15672/api/queues/' + vhost, auth=HTTPBasicAuth('name', 'password')).content
queues = json.loads(queues)
for queue in queues:
if queue:
name = queue.get("name")
if not name:
continue
command = cmd.replace('queue_name', name)
print command
os.system(command)
安装新版
vim /etc/yum.repos.d/rabbitmq-erlang.repo
# In /etc/yum.repos.d/rabbitmq-erlang.repo
[rabbitmq-erlang]
name=rabbitmq-erlang
baseurl=https://dl.bintray.com/rabbitmq/rpm/erlang/20/el/7
gpgcheck=1
gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc
repo_gpgcheck=0
enabled=1
下载rabbitmq
wget https://dl.bintray.com/rabbitmq/all/rabbitmq-server/3.7.5/rabbitmq-server-3.7.5-1.el7.noarch.rpm
yum install rabbitmq-server-3.7.5-1.el7.noarch.rpm
手工删除connection
rabbitmqctl list_connections pid port state user vhost host recv_cnt send_cnt send_pend name | awk '{if ($5=="vhost_celery_online") {print "rabbitmqctl close_connection "" $1 "" "manually closing idle connection"" | "/bin/bash"}}'
