高可用基础方案
环境安装
基于Linux Centos8、Docker、Docker Compose实现技术验证。
Docker
设置存储库
sudo yum install -y yum-utils
sudo yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
安装 Docker 引擎
sudo yum install docker-ce docker-ce-cli containerd.io
设置开机启动
systemctl enable docker.service
配置日志文件大小、阿里镜像
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://j3rmbyqj.mirror.aliyuncs.com"],
"log-driver":"json-file",
"log-opts": {"max-size":"500m", "max-file":"3"}
}
EOF
重载配置文件
sudo systemctl daemon-reload
重启Docker服务
sudo systemctl restart docker
设置存储位置（非必要步骤，系统磁盘空间较少时设置Docker存储空间）
创建数据卷目录
mkdir -p /home/docker/volume
编辑docker.service文件
vim /lib/systemd/system/docker.service
# ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --graph=/home/docker/volume
重载配置文件并重启服务
sudo systemctl daemon-reload
sudo systemctl restart docker
Docker Compose
下载安装文件
sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
授权安装文件
sudo chmod +x /usr/local/bin/docker-compose
建立软链接
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
验证
docker-compose --version
如果下载安装文件等待时间很长，可以下载离线文件上传至/usr/local/bin/目录
Nginx+Keepalived
| 服务器 | IP地址 | 安装软件 |
|---|---|---|
| nginx_master | 192.168.10.77 | docker+nginx+keepalived |
| nginx_backup | 192.168.10.79 | docker+nginx+keepalived |
| vip漂移地址 | 192.168.10.200 | keepalived.conf配置VIP地址 |
Nginx安装
创建Nginx启动脚本
version: '3'
services:
nginx:
image: nginx:latest
container_name: nginx
restart: always
privileged: true
ports:
- 8080:80
volumes:
- /home/docker/nginx/conf/nginx.conf:/etc/nginx/nginx.conf:ro
- /home/docker/nginx/conf/conf.d:/etc/nginx/conf.d
- /home/docker/nginx/log:/var/log/nginx
- /home/docker/nginx/html:/usr/share/nginx/html:ro
创建nginx.conf配置文件
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;
events {
use epoll; #Linux最常用支持大并发的事件触发机制
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
limit_conn_zone $binary_remote_addr zone=perip:10m; #添加limit_zone，限制同一IP并发数
include /etc/nginx/conf.d/*.conf;
}
nginx.conf文件可从docker容器内拷贝出来。
创建upstream代理配置文件host.conf网关节点ctc
upstream ctc {
ip_hash; #hash策略
server 192.168.10.192:13000 max_fails=1 fail_timeout=60s;
server 192.168.10.155:13000 max_fails=1 fail_timeout=60s;
}
节点代理指向后端网关服务。
创建前端代理文件ctc.conf
server {
listen 80;
server_name localhost;
location /api {
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://ctc;
}
# 查看nginx的并发连接数配置
location /NginxStatus {
stub_status on;
access_log off;
auth_basic "NginxStatus";
}
access_log off;
error_page 404 /404.html;
error_page 500 502 503 504 /404.html;
location = /404.html {
root html;
}
limit_conn perip 200; #同一ip并发数为200,超过会返回503
}
proxy_pass代理节点指向upstream配置。
执行脚本
docker-compose up -d
主备机脚本文件一致。
Keeppalived安装
下载安装包
https://www.keepalived.org/software/keepalived-2.0.7.tar.gz
更新依赖
yum install wget make gcc gcc-c++ openssl-devel
解压
tar zxvf keepalived-2.0.7.tar.gz
编译
cd keepalived-2.0.7
./configure --prefix=/home/keepalived
make
make install
删除安装包
rm -rf keepalived-2.0.7
rm -rf keepalived-2.0.7.tar.gz
配置服务启动
mkdir /etc/keepalived
cp /home/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
systemctl enable keepalived
创建nginx监听nginx_pid.sh脚本
#!/bin/bash
A=`ps -C nginx --no-header |wc -l`
if [ $A -eq 0 ];then
systemctl restart docker
sleep 3
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then
systemctl stop keepalived
fi
fi
脚本说明：当nginx进程不存在时，会自动重启docker服务，docker服务启动时会自动启动nginx容器；再次检查nginx进程，如果不存在，就停止keepalived服务，然后NGINX_BACKUP主机会自动接替NGINX_MASTER的工作。
脚本授权
chmod +x /etc/keepalived/nginx_pid.sh
配置/etc/keepalived/keepalived.conf启动文件
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
# 检查nginx状态的脚本
vrrp_script chk_nginx {
script "/etc/keepalived/nginx_pid.sh"
interval 2
weight 3
}
vrrp_instance VI_1 {
state MASTER #备份服务器上将MASTER改为BACKUP
interface ens37 # 网卡
virtual_router_id 51
priority 100 #备份服务上将100改为小于100，可配置成90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.200 #有多个vip可在下面继续增加
}
track_script {
chk_nginx
}
}
指定interface网卡与virtual_ipaddress漂移VIP地址
主备服务state 、priority节点配置
配置firewalld防火墙允许vrrp协议
firewall-cmd --permanent --add-rich-rule="rule family="ipv4" source address="192.168.10.79" protocol value="vrrp" accept"
firewall-cmd --reload
source address节点:主服务配置从IP，从服务配置主IP
在防火墙开启情况下进行该配置
启动和重启
systemctl start keepalived
systemctl restart keepalived
访问192.168.10.200:8080即可。
Nacos集群
构建脚本
services:
mysql:
image: mysql:5.7.24
container_name: mysql
restart: always
privileged: true
environment:
- "TZ=Asia/Shanghai"
- "MYSQL_ROOT_PASSWORD=123456"
ports:
- 3306:3306
volumes:
- /etc/localtime:/etc/localtime:ro
- /home/docker/mysql/conf/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
- /home/docker/mysql/data:/var/lib/mysql
nacos:
image: nacos/nacos-server:1.3.2
container_name: nacos-cluster-mysql
restart: always
privileged: true
environment:
- "MODE=cluster"
- "NACOS_SERVERS=192.168.10.77:8848,192.168.10.79:8848,192.168.10.26:8848"
- "NACOS_SERVER_IP=192.168.10.77"
- "SPRING_DATASOURCE_PLATFORM=mysql"
- "MYSQL_SERVICE_HOST=192.168.10.77"
- "MYSQL_SERVICE_DB_NAME=nacos"
- "MYSQL_SERVICE_PORT=3306"
- "MYSQL_SERVICE_USER=root"
- "MYSQL_SERVICE_PASSWORD=123456"
ports:
- 8848:8848
depends_on:
- mysql
可以写入nginx构建脚本
MODE nacos模式
NACOS_SERVERS 所有nacosIP端口地址
MYSQL_SERVICE_HOST 当前nacos对外暴露IP地址
MYSQL_SERVICE_DB_NAME当前nacos数据连接地址
数据库导入脚本文件
创建mysqld.cnf文件
[mysqld]
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
datadir = /var/lib/mysql
#log-error = /var/log/mysql/error.log
# By default we only accept connections from localhost
#bind-address = 127.0.0.1
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
lower_case_table_names=1
max_connections=1024
配置Nacos集群Nginx代理
创建upstream代理配置文件host.conf网关节点nacos
upstream nacos {
ip_hash; #hash策略
server 192.168.10.77:8848 max_fails=1 fail_timeout=60s;
server 192.168.10.79:8848 max_fails=1 fail_timeout=60s;
server 192.168.10.26:8848 max_fails=1 fail_timeout=60s;
}
节点代理指向Nacos服务。
创建nacos代理文件nacos.conf
server {
listen 8847;
server_name localhost;
location / {
proxy_pass http://nacos;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header REMOTE-HOST $remote_addr;
add_header X-Cache $upstream_cache_status;
add_header Cache-Control no-cache;
}
limit_conn perip 200; #同一ip并发数为200,超过会返回503
}
删除nginx容器并重新执行docker-compose编排文件
Redis哨兵
docker-compose.yml编排文件
version: '3'
services:
redis:
image: redis:4.0.14
container_name: redis
restart: always
privileged: true
command: redis-server /usr/local/etc/redis/redis.conf
volumes:
- /home/docker/redis/data:/data
- /home/docker/redis/conf/redis.conf:/usr/local/etc/redis/redis.conf
ports:
- 6379:6379
redis-sentinel:
image: bitnami/redis-sentinel:latest
container_name: redis-sentinel
restart: always
privileged: true
environment:
- REDIS_MASTER_HOST=192.168.10.77
- REDIS_MASTER_PASSWORD=123456
- REDIS_SENTINEL_PASSWORD=123456
ports:
- 26379:26379
REDIS_MASTER_HOST 主节点IP
REDIS_MASTER_PASSWORD 主节点密码
REDIS_SENTINEL_PASSWORD 哨兵密码
编辑redis.conf配置文件
bind 0.0.0.0
requirepass 123456
masterauth 123456
Master主节点的 redis.conf 配置
slaveof 192.168.10.77 6379
slave从节点的配置，在主节点配置基础上添加slaveof 节点信息
