keepalive +nginx实现高性能负载均衡

82次阅读

共计 6194 个字符，预计需要花费 16 分钟才能阅读完成。

导读	keepalived 的 HA 分为抢占模式和非抢占模式，抢占模式即 MASTER 从故障中恢复后，会将 VIP 从 BACKUP 节点中抢占过来。非抢占模式即 MASTER 恢复后不抢占 BACKUP 升级为 MASTER 后的 VIP

环境：cenos7 keepalive1.3.8 nginx1.12.2

vip 172.18.203.101 master 机器 nginx1: 外网 172.18.203.172 内网 172.18.1.172 slave 机器 nginx2：外网 172.18.203.173 内网 172.18.1.173

keepalive master 配置文件

[root@keepalive ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived #全局定义

global_defs {
notification_email {xiaofeng@sunspeedy.com}

notification_email_from xiaofeng@sunspeedy.com
smtp_server smtp.exmail.qq.com
smtp_connect_timeout 30
router_id master-node
}

vrrp_script chk_http_port {
script "/opt/chk_nginx.sh"
interval 2
weight -5
fall 2
rise 1
}

vrrp_instance VI_1 {
state MASTER
interface ens192
mcast_src_ip 172.18.203.172
unicast_peer {172.18.203.173 ##（对端 IP 地址）此地址一定不能忘记,vrrp need use}

virtual_router_id 51
priority 101
advert_int 1
authentication {
auth_type PASS #设置 vrrp 验证类型，主要有 PASS 和 AH 两种
auth_pass 1111
}
virtual_ipaddress { #VRRP HA 虚拟地址 如果有多个 VIP，继续换行填写
172.18.203.101
}

notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"

track_script {chk_http_port}
}

nginx 配置

[root@keepalive ~]# cat /usr/local/nginx/conf/nginx.conf
user nginx;
worker_processes 2;

#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;

#pid logs/nginx.pid;

events {worker_connections 65535;}

http {
include mime.types;
default_type application/octet-stream;
charset utf-8;

######
## set access log format
######
log_format main '$http_x_forwarded_for $remote_addr $remote_user [$time_local]"$request"''$status $body_bytes_sent "$http_referer" ''"$http_user_agent""$http_cookie" $host $request_time';

#######
## http setting
#######
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
proxy_cache_path /var/www/cache levels=1:2 keys_zone=mycache:20m max_size=2048m inactive=60m;
proxy_temp_path /var/www/cache/tmp;

fastcgi_connect_timeout 3000;
fastcgi_send_timeout 3000;
fastcgi_read_timeout 3000;
fastcgi_buffer_size 256k;
fastcgi_buffers 8 256k;
fastcgi_busy_buffers_size 256k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;

#
client_header_timeout 600s;
client_body_timeout 600s;
# client_max_body_size 50m;
client_max_body_size 100m; #允许客户端请求的最大单个文件字节数
client_body_buffer_size 256k; #缓冲区代理缓冲请求的最大字节数，可以理解为先保存到本地再传给用户

gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_comp_level 9;
gzip_types text/plain application/x-javascript text/css application/xml text/javascript application/x-httpd-php;
gzip_vary on;

## includes vhosts
include vhosts/*.conf;
}

[root@keepalive ~]# cat /usr/local/nginx/conf/vhosts/ntt52101.conf
upstream LB-WWW {
ip_hash;
server 172.18.1.155:52101 max_fails=3 fail_timeout=30s; #max_fails = 3 为允许失败的次数，默认值为 1
server 172.18.1.156:52101 max_fails=3 fail_timeout=30s; #fail_timeout = 30s 当 max_fails 次失败后，暂停将请求分发到该后端服务器的时间
}

server {
listen 52101;
###### 如果后端有多组 web，需要将其域名解析到 vip
server_name 172.18.203.101;
access_log /usr/local/nginx/logs/nttinterface_access.log main;
error_log /usr/local/nginx/logs/nttinterface_error.log;

location / {
proxy_pass http://LB-WWW;
proxy_redirect off ;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 300; #跟后端服务器连接超时时间，发起握手等候响应时间
proxy_send_timeout 300; #后端服务器回传时间，就是在规定时间内后端服务器必须传完所有数据
proxy_read_timeout 600; #连接成功后等待后端服务器的响应时间，已经进入后端的排队之中等候处理
proxy_buffer_size 256k; #代理请求缓冲区, 会保存用户的头信息以供 nginx 进行处理
proxy_buffers 4 256k; #同上，告诉 nginx 保存单个用几个 buffer 最大用多少空间
proxy_busy_buffers_size 256k; #如果系统很忙时候可以申请最大的 proxy_buffers
proxy_temp_file_write_size 256k; #proxy 缓存临时文件的大小
proxy_next_upstream error timeout invalid_header http_500 http_503 http_404;
proxy_max_temp_file_size 128m;
proxy_cache mycache;
proxy_cache_valid 200 302 60m;
proxy_cache_valid 404 1m;
}
}

slave 端

[root@keepalive src]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
notification_email {xiaofeng@sunspeedy.com}

notification_email_from xiaofeng@sunspeedy.com
smtp_server smtp.exmail.qq.com
smtp_connect_timeout 30
router_id slave-node
}

vrrp_script chk_http_port {
script "/opt/chk_nginx.sh"
interval 2
weight -5
fall 2
rise 1
}

vrrp_instance VI_1 {
state BACKUP
interface ens192
mcast_src_ip 172.18.203.173
unicast_peer {172.18.203.172 ##（对端 IP 地址）此地址一定不能忘记,vrrp need use}

virtual_router_id 51
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {172.18.203.101}

notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"

track_script {chk_http_port}

}

ngixn 检查

[root@keepalive src]# cat /opt/
chk_nginx.sh frp/
[root@keepalive src]# cat /opt/chk_nginx.sh
#!/bin/bash
counter=$(ps -C nginx --no-heading|wc -l)
if ["${counter}" = "0" ]; then
/usr/local/nginx/sbin/nginx
sleep 2
counter=$(ps -C nginx --no-heading|wc -l)
if ["${counter}" = "0" ]; then
/etc/init.d/keepalived stop
fi
fi

发送邮件

[root@keepalive src]# cat /etc/keepalived/notify.sh

#!/bin/bash
# Author: MageEdu <linuxedu@foxmail.com> # description: An example of notify script
#

vip=172.18.203.101
contact='xiaofeng@sunspeedy.com'

notify() {
mailsubject="`hostname` to be $1: $vip floating"
mailbody="`date'+%F %H:%M:%S'`: vrrp transition, `hostname` changed to be $1"
echo $mailbody | mail -s "$mailsubject" $contact
}

防火墙配置

1008 iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
1009 iptables -A INPUT -p icmp -j ACCEPT
1010 iptables -A INPUT -i lo -j ACCEPT
1011 iptables -A INPUT -s 172.18.203.0/24 -d 224.0.0.18 -j ACCEPT
1012 iptables -A INPUT -s 172.18.1.0/24 -d 224.0.0.18 -j ACCEPT
1013 iptables -A INPUT -s 172.18.203.0/24 -p vrrp -j ACCEPT
1014 iptables -A INPUT -s 172.18.1.0/24 -p vrrp -j ACCEPT
1015 iptables -A INPUT -p tcp -m multiport --dport 80,22,52101,8123 -j ACCEPT
1016 iptables -A INPUT -j REJECT --reject-with icmp-host-prohibited
1017 iptables -A FORWARD -j REJECT --reject-with icmp-host-prohibited
1018 iptables-save
1019 history

iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
1084 iptables -A INPUT -p icmp -j ACCEPT
1085 iptables -A INPUT -i lo -j ACCEPT
1086 iptables -A INPUT -d 172.18.203.101 -j ACCEPT
1087 iptables -A INPUT -s 172.18.203.0/24 -d 224.0.0.18 -j ACCEPT
1088 iptables -A INPUT -s 172.18.1.0/24 -d 224.0.0.18 -j ACCEPT
1089 iptables -A INPUT -s 172.18.203.0/24 -p vrrp -j ACCEPT
1090 iptables -A INPUT -s 172.18.1.0/24 -p vrrp -j ACCEPT
1091 iptables -A INPUT -p tcp -m multiport --dport 80,22,52101,8123 -j ACCEPT
1092 iptables -A INPUT -j REJECT --reject-with icmp-host-prohibited
1093 iptables -A FORWARD -j REJECT --reject-with icmp-host-prohibited
1094 iptables -L -n
1095 iptables-save