一、前言
实现基于keepalived+nginx+Varnish+LAMP企业架构，在后端LAMP部署wordpress站点，用nginx做反代，并且nginx反代服务器做keepalived主备架构；在nginx反代服务器后部署varnish缓存服务器，设置缓存规则，做压测测试检验效果。
实验架构图
二、环境准备
1、安装服务
在对应的服务器上安装相应服务程序包：
在RS1和RS2 上安装httpd服务：
~]#yum install -y httpd
在RS1和RS2上下载wordpress安装包：
~]# cd /var/www/html/
html]# wget https://cn.wordpress.org/wordpress-4.9.4-zh_CN.tar.gz
在RS1 上安装mariadb-server服务，让RS1提供数据库服务：
~]# yum install -y mariadb-server
在RS2上安装PHP-fpm的相关服务程序，让RS2负责动态内容的解析：
~]# yum install -y epel-release
~]# yum install -y php-fpm php-mysql php-mbstring php-mcrypt
在DR1和DR2 上安装keepalived和nginx服务：
~]#yum install -y epel-release
~]#yum install -y keepalived nginx
在DR1和DR2上安装psmisc服务，提供killall命令：
~]# yum install -y psmisc
在varnish上安装varnish服务：
~]#yum install -y epel-release
~]#yum install -y varnish
2、所有主机上同步ntp时间；清空防火墙，关闭selinux
~]# ntpdate ntp1.aliyun.com #同步时间
~]#iptable-F #清空防火墙
~]#setenforce 0 #临时关闭selinux
三、配置RS1和RS2
1、配置RS1
解压wordpress安装包并生成软连接并创建默认访问主页
[root@rs1 ~]# cd /var/www/html/
[root@rs1 ~]# tar xf wordpress-4.9.4-zh_CN.tar.gz
[root@rs1 ~]# ln -sv wordpress blog
[root@rs1 html]# vim index.html #创建默认主页
<h1>this is rs1</h1>
[root@rs1 html]# vim index.php #创建php主页
<?php
phpinfo();
?>
编辑创建httpd的vhost配置文件
[root@rs1 html]# vim /etc/httpd/conf.d/vhost.conf
<virtualhost *:80>
servername www.ilinux.io
DirectoryIndex index.html index.php
Documentroot /var/www/html
ProxyRequests off
ProxyPassMatch ^/(.*\.php)$ fcgi://192.168.32.137:9000/var/www/html/$1
ProxyPassMatch ^/(ping|status)$ fcgi://192.168.32.137:9000/$1
<Directory / >
options FollowSymlinks
Allowoverride none
Require all granted
</Directory>
</virtualhost>
配置完成后启动httpd服务
[root@rs1 html]# systemctl start httpd
接着编辑配置/etc/my.cnf数据库配置文件：
[root@rs1 html]# cd
[root@rs1 ~]# vim /etc/my.cnf
[mysqld]
skip_name_resolve=ON #添加此行
innodb_file_per_table=ON #添加此行
datadir=/var/lib/mysql
…………………………
启动mariadb-server服务并做安全配置
[root@rs1 ~]# systemctl start mariadb.service
[root@rs1 ~]# mysql_secure_installation
创建数据库wordpress并授权wordpress远程登录用户
[root@rs1 ~]# mysql -uroot -predhat
MariaDB [(none)]> CREATE DATABASE wordpress;
Query OK, 1 row affected (0.00 sec)
MariaDB [(none)]> GRANT ALL ON wordpress.* to 'wpuser'@'192.168.32.%' IDENTIFIED BY "magedu";
Query OK, 0 rows affected (0.00 sec)
MariaDB [(none)]> FLUSH PRIVILEGES;
Query OK, 0 rows affected (0.00 sec)
2、配置RS2
解压wordpress安装包并生成软连接并创建默认访问主页
[root@rs2 ~]# cd /var/www/html/
[root@rs2 ~]# tar xf wordpress-4.9.4-zh_CN.tar.gz
[root@rs2 ~]# ln -sv wordpress blog
[root@rs2 html]# vim index.html #创建默认主页
<h1>this is rs2</h1>
[root@rs2 html]# vim index.php #创建php主页
<?php
phpinfo();
?>
编辑创建httpd的vhost配置文件
[root@rs2 ~]# vim /etc/httpd/conf.d/vhost.conf
<virtualhost *:80>
servername www.ilinux.io
DirectoryIndex index.html index.php
Documentroot /var/www/html
ProxyRequests off
ProxyPassMatch ^/(.*\.php)$ fcgi://127.0.0.1:9000/var/www/html/$1
ProxyPassMatch ^/(ping|status)$ fcgi://127.0.0.1:9000/$1
<Directory / >
options FollowSymlinks
Allowoverride none
Require all granted
</Directory>
</virtualhost>
启动httpd服务
[root@rs2 ~]# systemctl start httpd
接着配置php-fpm，编辑文件/etc/php-fpm.d/www.conf
[root@rs2 ~]# vim /etc/php-fpm.d/www.conf
listen = 0.0.0.0:9000
;listen.allowed_clients = 127.0.0.1 #注释掉此句，默认允许Ip访问
pm.status_path = /status
ping.path = /ping
ping.response = pong
创建PHP会话目录，并修改目录的属主和属组：
[root@rs2 ~]# mkdir /var/lib/php/session/
[root@rs2 ~]# chown apache:apache /var/lib/php/session
启动php-fpm服务
[root@rs2 ~]# systemctl start php-fpm
四、配置DR1和DR2
1、配置DR1
编辑/etc/nginx/nginx.conf文件：
[root@dr1 ~]# vim /etc/nginx/nginx.conf
http {
......
upstream apservers {
server 192.168.32.136:80;
server 192.168.32.137:80;
server 127.0.0.1:80 backup; #在本地提供备用通知页面
}
......
server {
......
location / {
proxy_pass http://apservers;
proxy_set_header host $http_host;
proxy_set_header X-Forward-For $remote_addr;
}
......
}
更改本地的默认主页：
[root@dr1 ~]# vim /usr/share/nginx/html/index.html
<h1>under maintenanxe dr1</h1>
最后检查nginx配置无误后启动nginx服务：
[root@dr1 ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@dr1 ~]# nginx
接着配置keepalived服务：
[root@dr1 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id dr1
vrrp_mcast_group4 224.1.101.33
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_script chk_ngx { #用于检查在此服务器上的nginx进程是否存在，如果不存在则做减权，keepalived将做主备切换
script "killall -0 nginx && exit 0 || exit 1"
weight -10
interval 1
fall 3
rise 3
}
vrrp_instance VI_1 {
state MASTER #DR1作为keepalived的主机，因此设置state为MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
track_script {
chk_ngx
}
virtual_ipaddress {
192.168.0.99/24 dev ens37 label ens33:0
}
}
最后启动keepalived服务：
[root@dr1 ~]# systemctl start keepalived.service
- 配置DR2
同理配置/etc/nginx/nginx.conf，本地默认主页
配置keepalived服务如下：
[root@dr2 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id dr2
vrrp_mcast_group4 224.1.101.33
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_script chk_ngx {
script "killall -0 nginx && exit 0 || exit 1"
weight -10
interval 1
fall 3
rise 3
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 98
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
track_script {
chk_ngx
}
virtual_ipaddress {
192.168.0.99/24 dev ens37 label ens33:0
}
}
最后启动nginx、keepalived服务：
[root@dr2 ~]# nginx
[root@dr2 ~]# systemctl start keepalived.service
五、使用客户端压测
目前我们已经搭建好了keepalived+nginx反代+LAMP的环境，在配置varnish缓存前，我们先来测试下目前这种架构访问默认主页及wordpress页面的情况。
[root@wujunjie6 ~]# for i in {1..10}; do curl http://192.168.0.99 ; done
<h1>this is rs1</h1>
<h1>this is rs2</h1>
<h1>this is rs1</h1>
<h1>this is rs2</h1>
<h1>this is rs1</h1>
<h1>this is rs2</h1>
<h1>this is rs1</h1>
<h1>this is rs2</h1>
<h1>this is rs1</h1>
<h1>this is rs2</h1>
[root@wujunjie6 ~]# ab -c 200 -n 10000 http://192.168.0.99/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 192.168.0.99 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests
Server Software: nginx/1.12.2
Server Hostname: 192.168.0.99
Server Port: 80
Document Path: /
Document Length: 21 bytes
Concurrency Level: 200
Time taken for tests: 16.321 seconds
Complete requests: 10000
Failed requests: 0
Write errors: 0
Total transferred: 2720000 bytes
HTML transferred: 210000 bytes
Requests per second: 612.69 [#/sec] (mean)
Time per request: 326.427 [ms] (mean)
Time per request: 1.632 [ms] (mean, across all concurrent requests)
Transfer rate: 162.75 [Kbytes/sec] received
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 91 424.5 1 7004
Processing: 31 228 79.4 214 3070
Waiting: 7 227 79.4 213 3069
Total: 36 319 439.0 217 7409
Percentage of the requests served within a certain time (ms)
50% 217
66% 230
75% 239
80% 248
90% 323
95% 1204
98% 1255
99% 1818
100% 7409 (longest request)
从压测情况来说，访问默认主页的速度大概是在每秒600个情况。
那么接着我们来看看对访问wordpress进行压测。
[root@wujunjie6 ~]# ab -c 200 -n 10000 http://192.168.0.99/blog/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 192.168.0.99 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests
Server Software: nginx/1.12.2
Server Hostname: 192.168.0.99
Server Port: 80
Document Path: /blog/
Document Length: 0 bytes
Concurrency Level: 200
Time taken for tests: 166.943 seconds
Complete requests: 10000
Failed requests: 130
(Connect: 0, Receive: 0, Length: 130, Exceptions: 0)
Write errors: 0
Non-2xx responses: 10000
Total transferred: 2424746 bytes
HTML transferred: 33464 bytes
Requests per second: 59.90 [#/sec] (mean)
Time per request: 3338.856 [ms] (mean)
Time per request: 16.694 [ms] (mean, across all concurrent requests)
Transfer rate: 14.18 [Kbytes/sec] received
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 173 2266.6 0 31004
Processing: 3 3142 3204.8 2756 88271
Waiting: 3 3142 3204.8 2755 88271
Total: 3 3315 3914.4 2759 88272
Percentage of the requests served within a certain time (ms)
50% 2759
66% 2855
75% 2915
80% 2960
90% 3561
95% 4827
98% 17423
99% 25141
100% 88272 (longest request)
从压测情况来看，访问wordpress页面的速度大概在每秒50个，可想后端的虚拟机压力非常大
六、配置varnish代理缓存
1、配置varnish
首先修改/etc/varnish/varnish.params文件：
[root@varnish ~]# vim /etc/varnish/varnish.params
RELOAD_VCL=1
VARNISH_VCL_CONF=/etc/varnish/default.vcl #调用默认的vcl配置文件
VARNISH_LISTEN_PORT=6081 #varnish服务监听的端口
VARNISH_ADMIN_LISTEN_ADDRESS=10.10.10.13 #varnish监听的地址
VARNISH_ADMIN_LISTEN_PORT=6082 #varnish监听的管理端口
VARNISH_SECRET_FILE=/etc/varnish/secret #varnish管理接口的密钥文件
VARNISH_STORAGE="file,/data/cache,1G" #设置varnish缓存数据的方式为文件及缓存大小。
VARNISH_USER=varnish
VARNISH_GROUP=varnish
[root@varnish ~]# mkdir /data/cache #创建缓存路径
[root@varnish ~]# chown varnish:varnish /data/cache/ #更改路径属主属组
接着修改varnish的配置文件/etc/varnish/default.vcl，首先添加与调度后端服务器相关的配置
[root@varnish ~]# vim /etc/varnish/default.vcl
import directors; #加载调度相关的模块
probe backend_healthcheck { #定义后端服务器的健康检查机制
.url = "/index.html";
.window = 5;
.threshold = 4;
.interval = 2s;
.timeout = 1s;
}
backend RS1 { #定义后端服务器RS1并调用健康检查
.host = "192.168.32.136";
}
backend RS2 { #定义后端福气RS2并调用健康检查
.host = "192.168.32.137";
.port = "80";
.probe = backend_healthcheck;
}
sub vcl_init { #初始化创建后端服务器组
new BE = directors.round_robin();
BE.add_backend(RS1);
BE.add_backend(RS2);
}
acl purgers { #定义可访问的来源IP，用作权限
"127.0.0.1";
"192.168.32.0"/16;
}
接着我们来配置相应的vcl配置：
[root@varnish ~]# vim /etc/varnish/default.vcl
#继续添加下述vcl配置
sub vcl_recv {
if (req.method == "GET" && req.http.cookie) { # 带cookie首部的GET请求也缓存
return(hash);
}
if (req.url ~ "index.php") { #将index.php页面直接pass到后端服务器处理
return(pass);
}
if (req.method == "PURGE") { #定义用于清理缓存
if (client.ip ~ purgers) {
return(purge);
}
}
if (req.http.X-Forward-For) { #为发往后端主机的请求添加X-Forward-For首部
set req.http.X-Forward-For = req.http.X-Forward-For + "," + client.ip;
} else {
set req.http.X-Forward-For = client.ip;
}
set req.backend_hint = BE.backend(); #调用之前定义的后端服务器组，测试发现不调用的话，只能匹配第一个后端服务器
return(hash);
}
sub vcl_hash { #定义hash引擎，对请求的url做hash处理
hash_data(req.url);
}
sub vcl_backend_response { #自定义缓存文件的缓存时长
if (bereq.url ~ "\.(jpg|jpeg|gif|png)$") {
set beresp.ttl = 1d;
}
if (bereq.url ~ "\.(html|css|js)$") {
set beresp.ttl = 12h;
}
if (beresp.http.Set-Cookie) { #定义带Set-Cookie首部的后端响应不缓存，直接返回给客户端
set beresp.grace = 30m;
return(deliver);
}
}
sub vcl_deliver {
if (obj.hits > 0) { #为响应添加X-Cache首部，显示缓存是否命中
set resp.http.X-Cache = "HIT from " + server.ip;
} else {
set resp.http.X-Cache = "MISS";
}
}
最后启动varnish服务：
[root@varnish ~]# systemctl start varnish
2、修改DR1和DR2的nginx配置
在前面的配置中，我们在DR1和DR2中配置的负载均衡目标是后端的RS1和RS2的IP，因此此处需要将其更改为varnish监听的Ip和端口。
[root@dr1 ~]# vim /etc/nginx/nginx.conf
……………………
upstream apservers {
#server 192.168.32.136:80;
#server 192.168.32.137:80;
server 192.168.32.128:6081;
server 127.0.0.1:80 backup;
}
……………..
[root@dr1 ~]# nginx -s reload
七、测试
此时再次对默认主页进行压测测试：
[root@wujunjie6 ~]# ab -c 200 -n 10000 http://192.168.0.99/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 192.168.0.99 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests
Server Software: nginx/1.12.2
Server Hostname: 192.168.0.99
Server Port: 80
Document Path: /
Document Length: 21 bytes
Concurrency Level: 200
Time taken for tests: 10.239 seconds
Complete requests: 10000
Failed requests: 0
Write errors: 0
Total transferred: 3578296 bytes
HTML transferred: 210000 bytes
Requests per second: 976.70 [#/sec] (mean)
Time per request: 204.771 [ms] (mean)
Time per request: 1.024 [ms] (mean, across all concurrent requests)
Transfer rate: 341.30 [Kbytes/sec] received
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 52 332.6 1 7001
Processing: 18 145 86.6 131 3193
Waiting: 4 144 86.6 130 3193
Total: 29 197 365.2 132 7731
Percentage of the requests served within a certain time (ms)
50% 132
66% 142
75% 147
80% 152
90% 214
95% 332
98% 1148
99% 1358
100% 7731 (longest request)
分析默认主页的压测情况，在启动缓存后也有所提高，但是前后差距不大，可能是页面太小。
接着我们来尝试下压测wordpress页面：
[root@wujunjie6 ~]# ab -c 200 -n 10000 http://192.168.0.99/blog/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 192.168.0.99 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests
Server Software: nginx/1.12.2
Server Hostname: 192.168.0.99
Server Port: 80
Document Path: /blog/
Document Length: 0 bytes
Concurrency Level: 200
Time taken for tests: 10.509 seconds
Complete requests: 10000
Failed requests: 0
Write errors: 0
Non-2xx responses: 10000
Total transferred: 3487699 bytes
HTML transferred: 0 bytes
Requests per second: 951.54 [#/sec] (mean)
Time per request: 210.185 [ms] (mean)
Time per request: 1.051 [ms] (mean, across all concurrent requests)
Transfer rate: 324.09 [Kbytes/sec] received
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 54 401.8 0 7001
Processing: 8 148 92.3 119 3296
Waiting: 8 148 92.2 119 3296
Total: 49 202 420.6 121 7737
Percentage of the requests served within a certain time (ms)
50% 121
66% 127
75% 139
80% 202
90% 259
95% 323
98% 1120
99% 1318
100% 7737 (longest request)
从以上压测结果看，访问wordpress默认主页的速度有了大幅度提高，后端php-fpm主机的压力减少了许多。
