原理就这样了，现在开始部署
1，改机器名
[root@data-1-2 ~]# hostname
data-1-2
[root@data-1-2 ~]#
[root@data-1-1 ~]# hostname
data-1-1
[root@data-1-1 ~]#
[root@director ~]# hostname
director
[root@director ~]#
2，两个RS节点都安装http服务
[root@data-1-1 ~]# yum -y install httpd
3，启动http
4，改一下主页面
查看站点目录
[root@data-1-2 ~]# cat /etc/httpd/conf/httpd.conf | grep DocumentRoot
# DocumentRoot: The directory out of which you will serve your
DocumentRoot "/var/www/html"
# This should be changed to whatever you set DocumentRoot to.
# DocumentRoot /www/docs/dummy-host.example.com
[root@data-1-2 ~]# echo 133 > /var/www/html/index.html
[root@data-1-1 ~]# echo 128 > /var/www/html/index.html
5，安装lvs
[root@data-1-2 ~]# mkdir -p /home/oldboy/tools
[root@data-1-1 ~]# mkdir -p /home/oldboy/tools
[root@director ~]# mkdir -p /home/oldboy/tools
以后安装全部在director端
[root@director ~]# mkdir -p /home/oldboy/tools
[root@director ~]# cd /home/oldboy/tools
下载软件包
[root@director tools]# wget http://www.linuxvirtualserver.org/software/kernel-2.6/ipvsadm-1.24.tar.gz
--2016-04-14 02:23:51-- http://www.linuxvirtualserver.org/software/kernel-2.6/ipvsadm-1.24.tar.gz
Resolving www.linuxvirtualserver.org... 69.56.251.119, 2001:470:1f0f:297::2
Connecting to www.linuxvirtualserver.org|69.56.251.119|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 36598 (36K) [application/x-gzip]
Saving to: “ipvsadm-1.24.tar.gz”
100%[======================================>] 36,598 56.5K/s in 0.6s
2016-04-14 02:23:54 (56.5 KB/s) - “ipvsadm-1.24.tar.gz” saved [36598/36598]
查看内核模块
[root@director tools]#
[root@director tools]# lsmod | grep ip_vs
[root@director tools]# cat /etc/redhat-release
CentOS release 6.5 (Final)
[root@director tools]# uname -r
2.6.32-431.el6.i686
[root@director tools]#
[root@director tools]# ls -ld /usr/src/kernels/2.6.32-431.el6.i686/
drwxr-xr-x. 22 root root 4096 Apr 14 2016 /usr/src/kernels/2.6.32-431.el6.i686/
安装lvs需要对kernel目录作个软连接
[root@director tools]# ln -s /usr/src/kernels/2.6.32-431.el6.i686 /usr/src/linux
[root@director tools]# ls -l /usr/src/
total 8
drwxr-xr-x. 2 root root 4096 Sep 23 2011 debug
drwxr-xr-x. 3 root root 4096 Apr 14 2016 kernels
lrwxrwxrwx. 1 root root 36 Apr 14 02:34 linux -> /usr/src/kernels/2.6.32- 431.el6.i686
[root@director
解压软件包
[root@director tools]# tar -zxvf ipvsadm-1.24.tar.gz
[root@director tools]# ls
ipvsadm-1.24 ipvsadm-1.24.tar.gz
开始编译
[root@director tools]# cd ipvsadm-1.24
[root@director ipvsadm-1.24]# ls
config_stream.c dynamic_array.h ipvsadm-save libipvs VERSION
config_stream.h ipvsadm.8 ipvsadm-save.8 Makefile
contrib ipvsadm.c ipvsadm.sh README
debian ipvsadm-restore ipvsadm.spec SCHEDULERS
dynamic_array.c ipvsadm-restore.8 ipvsadm.spec.in TAGS
发现有makefiel直接make
Make
[root@director ipvsadm-1.24]# make install
此时检查一下内核模块
[root@director ipvsadm-1.24]# lsmod | grep ip_vs
[root@director ipvsadm-1.24]#
没有
执行ipvsadm
[root@director ipvsadm-1.24]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@director ipvsadm-1.24]#
也可以modprobe ip_vs加载到内核模块
再次检查内核模块
[root@director ipvsadm-1.24]# lsmod | grep ip_vs
ip_vs 102058 0
libcrc32c 841 1 ip_vs
ipv6 2610 18 ip_vs,ip6t_REJECT,nf_conntrack_ipv6,nf_defrag_ipv6
[root@director ipvsadm-1.24]#
配置lvs负载均衡服务
[root@localhost ~]# echo 'modprobe ip_vs' >> /etc/rc.d/rc.local
[root@localhost ~]# cat /etc/rc.d/rc.local
配置ip地址
[root@director ipvsadm-1.24]# ifconfig eth0:1 192.168.48.138 netmask 255.255.255.0 up
[root@localhost ~]# echo 'ifconfig eth0:1 192.168.48.138 netmask 255.255.255.0 up' >> /etc/rc.d/rc.local
下面是配好的IP
[root@localhost ~]# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:37:B7:13
inet addr:192.168.48.137 Bcast:192.168.48.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fe37:b713/ Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:619 errors:0 dropped:0 overruns:0 frame:0
TX packets:507 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:105788 (103.3 KiB) TX bytes:60062 (58.6 KiB)
Interrupt:19 Base address:0x2024
eth0:1 Link encap:Ethernet HWaddr 00:0C:29:37:B7:13
inet addr:192.168.48.138 Bcast:192.168.48.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Interrupt:19 Base address:0x2024
eth1 Link encap:Ethernet HWaddr 00:0C:29:37:B7:1D
inet addr:10.0.0.130 Bcast:10.0.0.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fe37:b71d/ Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:40 errors:0 dropped:0 overruns:0 frame:0
TX packets:11 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:5232 (5.1 KiB) TX bytes:1368 (1.3 KiB)
Interrupt:19 Base address:0x20a4
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:136 Metric:1
RX packets:16 errors:0 dropped:0 overruns:0 frame:0
TX packets:16 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:960 (960.0 b) TX bytes:960 (960.0 b)
检查一下能不能ping通vip
[root@data-1-1 network-scripts]# ping 192.168.48.138
PING 192.168.48.138 (192.168.48.138) 56(84) bytes of data.
bytes from 192.168.48.138: icmp_seq=1 ttl= time=1.42 ms
bytes from 192.168.48.138: icmp_seq=2 ttl= time=0.447 ms
bytes from 192.168.48.138: icmp_seq=3 ttl= time=0.661 ms
^C
--- 192.168.48.138 ping statistics ---
手工执行添加lvs服务
[root@localhost ~]# ipvsadm --set 30 5 60
[root@localhost ~]# ipvsadm -A -t 192.168.48.138:80 -s wrr -p 20
[root@localhost ~]# ipvsadm -a -t 192.168.48.138:80 -r 192.168.48.128:80 -g -w 1
[root@localhost ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.48.138:80 wrr persistent 20
-> 192.168.48.128:80 Route 1 0 0
[root@localhost ~]# ipvsadm -a -t 192.168.48.138:80 -r 192.168.48.133:80 -g -w 1
[root@localhost ~]#
[root@localhost ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.48.138:80 wrr persistent 20
-> 192.168.48.133:80 Route 1 0 0
-> 192.168.48.128:80 Route 1 0 0
[root@localhost ~]#
注意以下操作在real server上面执行
绑定rs vip
[root@data-1-1 CentOS_6.5_Final]# ifconfig lo:0 192.168.48.138 netmask 255.255.255.255up
[root@data-1-1 network-scripts]# route add -host 192.168.48.138 dev lo
[root@data-1-2 CentOS_6.5_Final]# ifconfig lo:0 192.168.48.138 netmask 255.255.255.255 up
[root@data-1-2 CentOS_6.5_Final]# route add -host 192.168.48.138 dev lo
现在手工抑制arp
[root@data-1-1 network-scripts]# echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
You have new mail in /var/spool/mail/root
[root@data-1-1 network-scripts]# echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
[root@data-1-1 network-scripts]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@data-1-1 network-scripts]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@data-1-2 CentOS_6.5_Final]# echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
[root@data-1-2 CentOS_6.5_Final]# echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
[root@data-1-2 CentOS_6.5_Final]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@data-1-2 CentOS_6.5_Final]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@data-1-2 CentOS_6.5_Final]#
现在访问一下试试
无法访问
[root@localhost ~]# service iptables stop
iptables: Setting chains to policy ACCEPT: filter [ OK ]
iptables: Flushing firewall rules: [ OK ]
iptables: Unloading modules: [ OK ]
[root@localhost ~]#
[root@localhost ~]# chkconfig iptables off
再次验证
[root@localhost ~]# ipvsadm -L --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Conns InPkts OutPkts InBytes OutBytes
-> RemoteAddress:Port
TCP 192.168.48.138:http 52 284 0 25611 0
-> 192.168.48.133:http 4 20 0 1323 0
-> 192.168.48.128:http 48 2 0 24288 0
下面是脚本配置lvs
Squid端
[root@localhost oldboy]# mv ipvs_server /usr/local/sbin/
[root@localhost oldboy]# cd /usr/local/sbin/
[root@localhost sbin]# chmod 700 ipvs_server
[root@localhost sbin]# cat ipvs_server
#!/bin/bash
. /etc/init.d/functions
VIP=192.168.48.138
GW=192.168.1.1
SUBNET=eth0:0
PORT=80
RIP1=(
192.168.48.128
)
RIP2=(
192.168.48.133
)
IFCONFIG=/sbin/ifconfig
ROUTE=/sbin/route
IPVSADM=/sbin/ipvsadm
ARPING=/sbin/arping
usage () {
local script_name
script_name=$1
echo 'Usage : $script_name [ start | stop | restart ]'
echo ""
return 1
}
checkCmd () {
if [ ! -f $1 ]; then
echo "cant find" $1
return 1
fi
}
checkSubnet () {
$IFCONFIG | grep "$1" |wc -l
}
ipvsStart () {
if
[ `checkSubnet $SUBNET` -ne 0 ]; then
$IFCONFIG $SUBNET down
fi
local rs
$IFCONFIG $SUBNET $VIP broadcast $VIP netmask 255.255.255.0 up
$IPVSADM -C
$IPVSADM -A -t $VIP:$PORT -s wrr -p 60
for ((i=0;i<`echo ${#RIP1[*]}`;i++))
do
$IPVSADM -a -t $VIP:$PORT -r ${RIP1[$i]}:$PORT -g -w 1
done
for ((i=0;i<`echo ${#RIP2[*]}`;i++))
do
$IPVSADM -a -t $VIP:$PORT -r ${RIP2[$i]}:$PORT -g -w 1
done
rs=$?
$IPVSADM >/tmp/oldboy.log
NetIf=`echo ${SUBNET}| awk -F ":" '{print $1}'`
$ARPING -c 1 -I ${NetIf} -s $VIP $GW >> /tmp/oldboy.log
[ $rs -eq 0 ] && action "Ipvsadm start." /bin/true
return $rs
}
ipvsStop () {
local rs
rs=1
$IFCONFIG $SUBNET down
$IPVSADM -C
$IPVSADM -Z
rs=$?
$ARPING -c 1 -I ${NetIf} -s $VIP $GW >/dev/null 2>&1
[ $rs -eq 0 ] && action "Ipvsadm stop." /bin/true
return $rs
}
main ()
{
if [ $# -ne 1 ];then
usage $0
fi
case "$1" in
start )
ipvsStart
;;
stop)
ipvsStop
;;
restart)
ipvsStop
ipvsStart
;;
*)
usage $0
;;
esac
}
main $*
Realserver端
两个realserver都一样
[root@data-1-1 ~]# cd /usr/local/sbin/
[root@data-1-1 sbin]# pwd
/usr/local/sbin
[root@data-1-1 sbin]# ls
rs_server
[root@data-1-1 sbin]# ll
total 4
-rwx------. 1 root root 1059 Apr 15 02:49 rs_server
[root@data-1-1 sbin]# chmod 700 rs_server
[root@data-1-1 sbin]# cat rs_server
#!/bin/bash
VIP=(
192.168.48.138
)
. /etc/rc.d/init.d/functions
case $1 in
start)
for ((i=0;i<`echo ${#VIP[*]}`;i++))
do
interface="lo:`echo ${VIP[$i]}| awk -F . '{print $4}'`"
/sbin/ifconfig $interface ${VIP[$i]} broadcast ${VIP[$i]} netmask 255.255.2 55.255 up
done
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
action "start lvs of realserver" /bin/true
;;
stop)
for ((i=0;i<`echo ${#VIP[*]}`;i++))
do
interface="lo:`echo ${VIP[$i]}| awk -F . '{print $4}'`"
/sbin/ifconfig $interface ${VIP[$i]} broadcast ${VIP[$i]} netmask 255.255.2 55.255 down
done
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
action "stop lvs of realserver" /bin/true
;;
*)
echo "usage: $0 {start|stop}"
exit 1
esac
[root@data-1-1 sbin]#
[root@data-1-1 ~]# rs_server stop
stop lvs of realserver [ OK ]
[root@data-1-1 ~]# rs_server start
start lvs of realserver [ OK ]
[root@data-1-1 ~]#
[root@data-1-1 ~]# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:5F:1C:77
inet addr:192.168.48.128 Bcast:192.168.48.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fe5f:1c77/ Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:448144 errors:0 dropped:0 overruns:0 frame:0
TX packets:901345 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:103293994 (98.5 MiB) TX bytes:1212121651 (1.1 GiB)
Interrupt:19 Base address:0x2000
eth1 Link encap:Ethernet HWaddr 00:0C:29:5F:1C:81
inet addr:10.0.0.128 Bcast:10.0.0.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fe5f:1c81/ Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:91636 errors:0 dropped:0 overruns:0 frame:0
TX packets:42 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:22347817 (21.3 MiB) TX bytes:3431 (3.3 KiB)
Interrupt:19 Base address:0x2080
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:374 errors:0 dropped:0 overruns:0 frame:0
TX packets:374 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:102744 (100.3 KiB) TX bytes:102744 (100.3 KiB)
lo:138 Link encap:Local Loopback
inet addr:192.168.48.138 Mask:255.255.255.255
UP LOOPBACK RUNNING MTU:65536 Metric:1
[root@data-1-1 ~]#
测试
现在写一个脚本进行健康检查做到如下，每10秒钟扫描一次realserver，当发现realserver宕机就自动从director删除，等恢复好再自动加回来
脚本如下可以放在后台执行
[root@localhost sbin]# cat health_check
#!/bin/bash
VIP="192.168.48.138"
port=80
RIP=(
192.168.48.128
192.168.48.133
)
while true
do
for ((i=0;i<`echo ${#RIP[*]}`;i++))
do
PORT_COUNT=`nmap ${RIP[$i]} -p $port|grep open |wc -l`
[ $PORT_COUNT -ne 1 ] && {
ipvsadm -d -t $VIP:$port -r ${RIP[$i]}:$port
echo "${RIP[$i]}" >> /tmp/rs.log
}
done
for j in `cat /tmp/rs.log`
do
PORT_COUNT=`nmap $j -p $port|grep open |wc -l`
[ $PORT_COUNT -eq 1 ] && {
ipvsadm -a -t $VIP:$port -r $j:$port -g -w 1
sed -i s#$j#""#g /tmp/rs.log
}
done
sleep 10
done
下面是网上下载的配置脚本
http://bbs.chinaunix.net/tree/index_294_1/
作者:NetSeek
http://www.linuxtone.org(IT
运维专家网|集群架构|性能调优)
欢迎转载,转载时请务必以超链接形式标明文章原始出处和作者信息及本声明.
更新时间:2008-10-27
背景:
随着你的网站业务量的增长你网站的服务器压力越来越大？需要
负载均衡
方案！商业的硬件如F5又太贵，你们又是创业型互联公司如何有效节约成本，节省不必要的浪费？同时
实现
商业硬件一样的高性能高可用的功能？有什么好的负载均衡可伸张可扩展的方案吗？答案是肯定的！有！我们利用LVS+Keepalived基于完整开源软件的架构可以为你提供一个负载均衡及高可用的服务器。
LVS+Keepalived 介绍
LVS
LVS是Linux Virtual Server的简写，意即Linux虚拟服务器，是一个虚拟的服务器集群系统。本项目在1998年5月由章文嵩博士成立，是中国国内最早出现的自由软件项目之一。目前有三种IP负载均衡技术（VS/NAT、VS/TUN和VS/DR）；
八种调度
算法
（rr,wrr,lc,wlc,lblc,lblcr,dh,sh）。
Keepalvied
Keepalived
在这里主要用作RealServer的健康状态检查以及LoadBalance主机和BackUP主机之间failover的实现
二. 网站负载均衡拓朴图
IP信息列表：
名称 IP
LVS-DR-Master 61.1.122.6
LVS-DR-BACKUP 61.1.122.7
LVS-DR-VIP 61.1.122.8
WEB1-Realserver 61.1.122.9
WEB2-Realserver 61.1.122.10
GateWay 61.1.122.1
三. 安装LVS和Keepalvied软件包
1.
下载
相关软件包
#mkdir /usr/local/src/lvs
#cd /usr/local/src/lvs
#wget
http://www.linuxvirtualserver.org/software/kernel-2.6/ipvsadm-1.24.tar.gz
#wget
http://www.keepalived.org/software/keepalived-1.1.15.tar.gz
2. 安装LVS和Keepalived
#lsmod |grep ip_vs
#uname -r
2.6.18-53.el5PAE
#ln -s /usr/src/kernels/2.6.18-53.el5PAE-i686/ /usr/src/linux
#tar zxvf ipvsadm-1.24.tar.gz
#cd ipvsadm-1.24
#make && make install
#find / -name ipvsadm # 查看ipvsadm的位置
#tar zxvf keepalived-1.1.15.tar.gz
#cd keepalived-1.1.15
#./configure && make && make install
#find / -name keepalived # 查看keepalived位置
#cp /usr/local/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/
#cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/
#mkdir /etc/keepalived
#cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/
#cp /usr/local/sbin/keepalived /usr/sbin/
#service keepalived start|stop #做成系统启动服务方便管理.
四. 配置LVS实现负载均衡
1． LVS-DR，配置LVS脚本实现负载均衡
#vi /usr/local/sbin/lvs-dr.sh
#!/bin/bash
# description: start LVS of DirectorServer
#Written by :NetSeek http://www.linuxtone.org
GW=61.1.122.1
# website director vip.
SNS_VIP=61.1.122.8
SNS_RIP1=61.1.122.9
SNS_RIP2=61.1.122.10
./etc/rc.d/init.d/functions
logger $0 called with $1
case "$1" in
start)
# set
squid
vip
/sbin/ipvsadm --set 30 5 60
/sbin/ifconfig eth0:0 $SNS_VIP broadcast $SNS_VIP netmask 255.255.255.255
broadcast $SNS_VIP up
/sbin/route add -host $SNS_VIP dev eth0:0
/sbin/ipvsadm -A -t $SNS_VIP:80 -s wrr -p 3
/sbin/ipvsadm -a -t $SNS_VIP:80 -r $SNS_RIP1:80 -g -w 1
/sbin/ipvsadm -a -t $SNS_VIP:80 -r $SNS_RIP2:80 -g -w 1
touch /var/lock/subsys/ipvsadm >/dev/null 2>&1
;;
stop)
/sbin/ipvsadm -C
/sbin/ipvsadm -Z
ifconfig eth0:0 down
ifconfig eth0:1 down
route del $SNS_VIP
route del $SS_VIP
rm -rf /var/lock/subsys/ipvsadm >/dev/null 2>&1
echo "ipvsadm stoped"
;;
status)
if [ ! -e /var/lock/subsys/ipvsadm ];then
echo "ipvsadm stoped"
exit 1
else
echo "ipvsadm OK"
fi
;;
*)
echo "Usage: $0 {start|stop|status}"
exit 1
esac
exit 0
2．配置Realserver脚本.
#vi /usr/local/sbin/realserver.sh
#!/bin/bash
# description: Config realserver lo and apply noarp
#Written by :NetSeek http://www.linuxtone.org
SNS_VIP=61.1.122.8
. /etc/rc.d/init.d/functions
case "$1" in
start)
ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
/sbin/route add -host $SNS_VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl-p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $SNS_VIP >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
exit 0
或者采用secondary ip address方式配置
# vi /etc/sysctl.conf
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
[code]
#sysctl –p
#ip addr add 61.1.122.8/32 dev lo
#ip add list 查看是否绑定
3. 启动lvs-dr脚本和realserver启本，在DR上可以查看LVS当前状态:
#watch ipvsadm –ln
五．利用Keepalvied实现负载均衡和和高可用性
1.配置在主负载均衡服务器上配置keepalived.conf
#vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
cnseek@gmail.com
# failover@firewall.loc
# sysadmin@firewall.loc
}
notification_email_from sns-lvs@gmail.com
smtp_server 127.0.0.1
# smtp_connect_timeout 30
router_id LVS_DEVEL
}
# 20081013 written by :netseek
# VIP1
vrrp_instance VI_1 {
state MASTER #备份服务器上将MASTER改为BACKUP
interface eth0
virtual_router_id 51
priority 100 #
备份
服务上将100改为99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
61.1.122.8
#(如果有多个VIP，继续换行填写.)
}
}
virtual_server 61.1.122.8 80 {
delay_loop 6 #(每隔10秒查询realserver状态)
lb_algo wrr #(lvs 算法)
lb_kind DR #(Direct Route)
persistence_timeout 60 #(同一IP的连接60秒内被分配到同一台realserver)
protocol TCP #(用TCP协议检查realserver状态)
real_server 61.1.122.9 80 {
weight 3 #(权重)
TCP_CHECK {
connect_timeout 10 #(10秒无响应超时)
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 61.1.122.10 80 {
weight 3
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
2. BACKUP服务器同上配置，先安装lvs再按装keepalived,仍后配置/etc/keepalived/keepalived.conf，只需将红色标示的部分改一下即可.
3. vi /etc/rc.local
#/usr/local/sbin/lvs-dr.sh 将lvs-dr.sh这个
脚本
注释掉。步
#/usr/local/sbin/lvs-dr.sh stop 停止lvs-dr脚本
#/etc/init.d/keepalived start 启动keepalived 服务，keepalived就能利用keepalived.conf 配
置
文件
，实现负载均衡和高可用.
4. 查看lvs服务是否正常
#watch ipvsadm –ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 61.1.122.8:80 wrr persistent 60
-> 61.1.122.10:80 Route 3 0 0
-> 61.1.122.9:80 Route 3 0 0
#tail –f /var/log/message 监听日志，查看状态，测试LVS负载均衡及高可用性是否有效。
5．停Master服务器的keepalived服务，查看BAKCUP服务器是否能正常接管服务。
四．相关参考
1．LVS 基础知识汇总
LVS的算法介绍
http://www.linuxtone.org/viewthread.php?tid=69
学习LVS的三种转发模式
http://www.linuxtone.org/viewthread.php?tid=77
LVS中的IP负载均衡技术
http://www.linuxtone.org/viewthread.php?tid=68
更多的请到
http://www.linuxtone.org
负载均衡版查看
Keepalived 相关参考资料。
http://www.keepalived.org/documentation.html
Lvs到此结束
下面是keepalive
下面是keepalived的正常工作和failover工作原理图
下面开始安装keepalived
准备两台主机
下载安装包以下操作两台机器同时操作
[root@localhost tools]# wget http://www.keepalived.org/software/keepalived-1.1.17.tar.gz
[root@localhost tools]# ln -s /usr/src/kernels/2.6.32-431.el6.i686/ /usr/src/linux
[root@localhost tools]# ls -l /usr/src/
total 8
drwxr-xr-x. 2 root root 4096 Sep 23 2011 debug
drwxr-xr-x. 3 root root 4096 Apr 14 09:48 kernels
lrwxrwxrwx. 1 root root 36 Apr 14 02:34 linux -> /usr/src/kernels/2.6.32-431.el6.i686
[root@localhost tools]#
解压
[root@director2 tools]# tar -zxvf keepalived-1.1.17.tar.gz
编译
[root@director2 tools]# cd keepalived-1.1.17
[root@localhost keepalived-1.1.17]# ./configure
configure: error:
!!! OpenSSL is not properly installed on your system. !!!
!!! Can not include OpenSSL headers files. !!!
报错了
yum install -y openssl openssl-devel
其它经验
其实系统安装了openssl，ssl.h文件也没问题，问题在于redhat AS3 把Kerberos includes 放在一个“奇怪”的地方，以致于keepalived的configure程序找不到，只好报openssl安装有问题。解决方法是设置编译环境的CPPFLAGS变量:export CPPFLAGS=-I/usr/kerberos/include ，再次编译就通过了。
再重新编译
[root@localhost keepalived-1.1.17]# ./configure
configure: error: Popt libraries is required
又报错
yum install popt-devel
再重新编译
[root@localhost keepalived-1.1.17]# ./configure
Keepalived configuration
------------------------
Keepalived version : 1.1.17
Compiler : gcc
Compiler flags : -g -O2
Extra Lib : -lpopt -lssl -lcrypto
Use IPVS Framework : Yes
IPVS sync daemon support : Yes
Use VRRP Framework : Yes
Use LinkWatch : No
Use Debug flags : No
三个yes就正确了1使用ipvs框架 2，支持ipvs的sync功能，3使用vrrp框架
[root@localhost keepalived-1.1.17]# make
[root@director2 keepalived-1.1.17]# make install
如果前面不做软连接可以用下面方法安装
[root@director2 keepalived-1.1.17]# cp /usr/local/etc/rc.d/init.d/keepalived /etc/init.d
[root@director2 keepalived-1.1.17]# cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/
[root@director2 keepalived-1.1.17]# mkdir /etc/keepalived
[root@director2 keepalived-1.1.17]# cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/
[root@director2 keepalived-1.1.17]# cp /usr/local/sbin/keepalived /usr/sbin/
[root@director2 keepalived-1.1.17]# service keepalived start
Starting keepalived: [ OK ]
[root@director2 keepalived-1.1.17]#
[root@localhost keepalived-1.1.17]# ps -ef | grep keepalived
root 5776 1 0 20:29 ? 00:00:00 keepalived -D
root 5778 5776 0 20:29 ? 00:00:00 keepalived -D
root 5779 5776 0 20:29 ? 00:00:00 keepalived -D
root 57 1260 0 20:30 pts/1 00:00:00 grep keepalive
配置文件说明
这里又是一个实例
下面标红的三行就是主备keepalived的配置文件区别
双实例互相间接管
单实例差异
现在编辑keepalived.conf
[root@localhost keepalived-1.1.17]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 150
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.48.138
}
}
备节点配置
[root@director2 keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.48.138
}
}
现在启动keepalived
[root@localhost keepalived]# service keepalived stop
Stopping keepalived: [ OK ]
[root@localhost keepalived]# service keepalived start
Starting keepalived:
主节点 [ OK ]
[root@director2 keepalived]# ip add |grep 192
inet 192.168.48.139/24 brd 192.168.48.255 scope global eth0
inet 192.168.48.138/32 scope global eth0
备节点
现在停掉主节点看看备节点能不能接管
[root@localhost keepalived]# service keepalived stop
Stopping keepalived: [ OK ]
[root@localhost keepalived]# ip add |grep 192
inet 192.168.48.137/24 brd 192.168.48.255 scope global eth0
[root@localhost keepalived]#
查看备节点
启动主节点
发现不能接管回来怀疑可能是防火墙的原因
[root@director2 keepalived]# service iptables stop
iptables: Setting chains to policy ACCEPT: filter [ OK ]
iptables: Flushing firewall rules: [ OK ]
iptables: Unloading modules: [ OK ]
[root@director2 keepalived]# chkconfig iptables off
[root@director2 keepalived]#
两边都有了
检查发现配置文件错误
priority 100
这里一样了
改正后
现在关闭主节点
看备节点
启动主节点
看备节点、
完全正常了
下面是切换过程中对外界ping vip的反应
可以配置多个vip
多加几行就可以
多实例就复制下面这段
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.48.138
}
}
瞎改，第一个实例是备，第二个改成实例主，改一下virtual_router_id
priority 100
virtual_ipaddress {
192.168.48.138
}
这样两边就是双主模式
演示通过keepalived实现httpd服务的高可用
两边都安装httpd
yum -y install httpd
[root@director ~]# service httpd start
Starting httpd: [ OK ]
[root@director ~]# echo 137 > /var/www/html/index.html
[root@director2 ~]# echo 139 > /var/www/html/index.html
现在让主节点宕机
正常接管业务
再启动
接管回来
高可用了
这种方式的高可用必须只能有程序不能有资源，资源放在共享存储
或者两台机器作实时同步
日志为/var/log/message
默认只能实现服务器级别的高可用要实现服务级别的高可用必须写脚本实现
下面是完整的keepalived+lvs的实际应用配置过程总共要4台服务器两台负载均衡器两台realserver
前面几步已经在上午安装完成
下面这步上午没做，需要两个director都执行
[root@director2 ipvsadm-1.24]# sed -i 's#net.ipv4.ip_forward = 0#net.ipv4.ip_forward = 1#' /etc/sysctl.conf
[root@director2 ipvsadm-1.24]# grep net.ipv4.ip_forward /etc/sysctl.conf
net.ipv4.ip_forward = 1
[root@director sbin]# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
error: "net.bridge.bridge-nf-call-ip6tables" is an unknown key
error: "net.bridge.bridge-nf-call-iptables" is an unknown key
error: "net.bridge.bridge-nf-call-arptables" is an unknown key
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 4294967295
kernel.shmall = 268435456
[root@director sbin]#
下面是全局块
其中下面这个是lvs标示要唯一
下面这个是vrrp实例组，在failover时以组为单位，要么同时切换要么同时不切换
下面是重点了
现在配置keepalived让keepalived来管理lvs和realserver
Master端
[root@director keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
lvs_sync_deamon_inteface eth0
virtual_router_id 51
priority 150
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.48.138
}
}
virtual_server 192.168.48.138 {
delay_loop 6
lb_algo wrr
lb_kind DR
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
real_server 192.168.48.128 80 {
weight 1
TCP_CHECK {
connect_timeout 8
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.48.133 80 {
weight 1
TCP_CHECK {
connect_timeout 8
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
[root@director keepalived]#
Backup端
[root@director keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
lvs_sync_deamon_inteface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.48.138
}
}
virtual_server 192.168.48.138 {
delay_loop 6
lb_algo wrr
lb_kind DR
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
real_server 192.168.48.128 80 {
weight 1
TCP_CHECK {
connect_timeout 8
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.48.133 80 {
weight 1
TCP_CHECK {
connect_timeout 8
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
[root@director keepalived]#
现在启动keepalived
master端状态
[root@director keepalived]# service keepalived start
Starting keepalived: [ OK ]
[root@director keepalived]# ip add | grep 192
inet 192.168.48.137/24 brd 192.168.48.255 scope global eth0
inet 192.168.48.138/32 scope global eth0
[root@director keepalived]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.48.138:0 wrr persistent 50
-> 192.168.48.133:80 Route 1 0 0
-> 192.168.48.128:80 Route 1 0 0
[root@director keepalived]#
Backup端状态
[root@director2 keepalived]# service keepalived start
Starting keepalived:
[root@director2 keepalived]# ip add | grep 192
inet 192.168.48.139/24 brd 192.168.48.255 scope global eth0
[root@director2 keepalived]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@director2 keepalived]#
配置realserver端
两个realserver都一样
[root@data-1-1 ~]# cd /usr/local/sbin/
[root@data-1-1 sbin]# pwd
/usr/local/sbin
[root@data-1-1 sbin]# ls
rs_server
[root@data-1-1 sbin]# ll
total 4
-rwx------. 1 root root 1059 Apr 15 02:49 rs_server
[root@data-1-1 sbin]# chmod 700 rs_server
[root@data-1-1 sbin]# cat rs_server
#!/bin/bash
VIP=(
192.168.48.138
)
. /etc/rc.d/init.d/functions
case $1 in
start)
for ((i=0;i<`echo ${#VIP[*]}`;i++))
do
interface="lo:`echo ${VIP[$i]}| awk -F . '{print $4}'`"
/sbin/ifconfig $interface ${VIP[$i]} broadcast ${VIP[$i]} netmask 255.255.2 55.255 up
done
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
action "start lvs of realserver" /bin/true
;;
stop)
for ((i=0;i<`echo ${#VIP[*]}`;i++))
do
interface="lo:`echo ${VIP[$i]}| awk -F . '{print $4}'`"
/sbin/ifconfig $interface ${VIP[$i]} broadcast ${VIP[$i]} netmask 255.255.2 55.255 down
done
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
action "stop lvs of realserver" /bin/true
;;
*)
echo "usage: $0 {start|stop}"
exit 1
esac
访问一下试试
现在模拟一台realserver宕机
检查负载均衡器状态
发现自动剔除了
现在访问一下页面
能访问，高可用了
看看backup端
Backup端也一样自动剔除了发现backup端也有转发列表
再检查vip
发现backup端没有vip，所以事实上backup端是无法转发的
现在把宕机的realserver启动
检查master端状态
检查backup端状态
两端全部已经接管回来了
访问网页试试
很正常
现在模拟master端宕机
检查backup端状态
很正常接管了
访问一下页面
页面不受影响
现在启动master端看看能不能接管回来
检查master端状态
检查backup端状态
正常接管回来了
到此全部结束
可以在/etc/rc.local里面加入
/usr/local/sbin/rs_server start语句开机自动设置realserver
下面是生产维护要点
、
