在DOCKER 环境下使用keepalived+Nginx实现高可用

1、环境准备

安装docker-desktop

2、制作keepalived+Nginx 的image

2.1 编制并启动基础版本的centos

docker-compose.yml 文件如下：

version: '3.9'
services:
	keepalivebase:
        image: 'centos:8'
        container_name: 'keepalivebase'
        privileged: true
        entrypoint: "/sbin/init"
        stdin_open: true
        tty: true
        networks:
            keepalive-ha:
                ipv4_address: '172.29.0.11'
      portainer:
        image: portainer/portainer:latest
        container_name: portainer
        restart: always
        environment:
            TZ: Asia/Shanghai
            LANG: en_US.UTF-8
        ports:
            - "9000:9000"
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock
        networks:
            keepalive-ha:
                ipv4_address: '172.29.0.50'
    networks:
	    keepalive-ha:
	        name: keepalive-ha
	        driver: bridge
	        # driver: overlay
	        ipam:
	            config:
	                - subnet: '172.29.0.0/16'

2.2 搭建keepalived,Nginx 环境

进入docker-compose.yml目录，执行

docker-compose up

安装工具

yum install -y net-tools
yum install -y psmisc
yum install rsyslog

安装keepalived

yum list keepalived
yum install -y keepalived

安装nginx

yum install -y nginx

修改 keepalived 启动参数

vi /etc/sysconfig/keepalived
把启动参数修改为
KEEPALIVED_OPTIONS="-D -d -S 0" 

修改日志输出配置

vi /etc/rsyslog.conf    
增加内容
local0.* /var/log/keepalived.log

2.3验证环境

重启动日志服务

systemctl restart rsyslog

启动 nginx服务,并且启动正常

systemctl start nginx

启动 keepalived服务,并且启动正常

systemctl start keepalived

设置服务自启动

systemctl enable nginx
systemctl enable keepalived

2.4 保存docker image

查询当前运行的docker

docker ps

保存docker image

docker  commit -a "zwwang" -m "centos7+keepalived+nginx" 424a09930292    centos8_keepalived_nginx:v1

3、编排容器以及编制配置文件

3.1 编制docker-compose

version: '3.9'
services:  
    keepalive01:
        image: 'centos8_keepalived_nginx:v1'
        container_name: 'keepalive01'
        privileged: true
        entrypoint: "/sbin/init"
        environment:
            - TZ=Asia/Shanghai
        volumes:
            - ./keepalived_master.conf:/etc/keepalived/keepalived.conf
            - ./backup.sh:/etc/keepalived/backup.sh
            - ./fault.sh:/etc/keepalived/fault.sh
            - ./master.sh:/etc/keepalived/master.sh
            - ./index-master.html:/usr/share/nginx/html/index.html
        command: /bin/bash -c "chmod +x /etc/keepalived/*.sh && chmod -x /etc/keepalived/keepalived.conf"
        ports:
            - "80:80"
        networks:
            keepalive-ha:
                ipv4_address: '172.29.0.11'
    keepalive02:
        image: 'centos8_keepalived_nginx:v1'
        container_name: 'keepalive02'
        privileged: true
        environment:
            - TZ=Asia/Shanghai
        entrypoint: "/sbin/init"
        volumes:
            - ./keepalived_buckup.conf:/etc/keepalived/keepalived.conf
            - ./backup.sh:/etc/keepalived/backup.sh
            - ./fault.sh:/etc/keepalived/fault.sh
            - ./master.sh:/etc/keepalived/master.sh
            - ./index-slave.html:/usr/share/nginx/html/index.html
        command: /bin/bash -c "chmod +x /etc/keepalived/*.sh && chmod -x /etc/keepalived/keepalived.conf"
        ports:
            - "81:80"
        networks:
            keepalive-ha:
                ipv4_address: '172.29.0.12'
    cul_test:
        image: 'centos:8'
        container_name: 'cul_test'
        privileged: true
        environment:
            - TZ=Asia/Shanghai
        entrypoint: "/sbin/init"
        stdin_open: true
        tty: true
        networks:
            keepalive-ha:
                ipv4_address: '172.29.0.13'
    portainer:
        image: portainer/portainer:latest
        container_name: portainer
        restart: always
        environment:
            TZ: Asia/Shanghai
            LANG: en_US.UTF-8
        ports:
            - "9000:9000"
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock
        networks:
            keepalive-ha:
                ipv4_address: '172.29.0.50'

networks:
    keepalive-ha:
        name: keepalive-ha
        driver: bridge
        # driver: overlay
        ipam:
            config:
                - subnet: '172.29.0.0/16'

3.2 编制keepalived 配置文件

3.2.1 主机配置文件

文件名为keepalived_master.conf

! Configuration File for keepalived
global_defs {
   notification_email {
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 182.254.38.18 
   smtp_connect_timeout 30
   router_id study01
    # VRRP的ipv4和ipv6的广播地址，配置了VIP的网卡向这个地址广播来宣告自己的配置信息，下面是默认值
#    # vrrp_mcast_group4 224.0.0.18
#    # vrrp_mcast_group6 ff02::12
}
vrrp_script check_httpd {
    script "/usr/bin/killall -0 nginx"
    interval 2
}
vrrp_instance VI_1 {
    # 定义初始状态，可以是MASTER或者BACKUP
    state MASTER 
    # 工作接口，通告选举使用哪个接口进行
    interface eth0
    # ID还是虚拟MAC最后一段地址的信息，取值范围0-255
    virtual_router_id 51
    # 使用哪个虚拟MAC地址
    # use_vmac XX:XX:XX:XX:XX

    # 监控本机上的哪个网卡，网卡一旦故障则需要把VIP转移出去
    track_interface {
        eth0
        # ens33
    }
    # 如果你上面定义了MASTER,这里的优先级就需要定义的比其他的高
    priority 100
    # 采用单播方式，指定本机和备机ip
    # unicast_src_ip 10.244.97.4 # local-IP
    # unicast_peer {
    #     10.244.97.5 # peer-IP
    # }
    # 通告频率，单位为秒
    advert_int 1
    # advert_int 0.5
    # 通信认证机制，这里是明文认证还有一种是加密认证
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # 三个指令，如果主机状态变成Master|Backup|Fault之后会去执行的通知脚本，脚本要自己写
    notify_master "/etc/keepalived/master.sh" 
    notify_backup "/etc/keepalived/backup.sh"
    notify_fault "/etc/keepalived/fault.sh"
    track_script {
        check_httpd 
    }
    virtual_ipaddress {
        # 192.168.31.9 
        # IP/掩码 dev 配置在哪个网卡
        # 192.168.200.16/24 dev eth1
        # IP/掩码 dev 配置在哪个网卡的哪个别名上
        172.29.0.20/24 dev eth0 label eth0:0 
    }
    # 虚拟路由，在需要的情况下可以设置lvs主机 数据包在哪个网卡进来从哪个网卡出去    
    # virtual_routes {
    #     192.168.110.0/24 dev eth2
    # }
    # 工作模式，nopreempt表示工作在非抢占模式，默认是抢占模式 preempt
    # nopreempt|preempt
    # 如果是抢占默认则可以设置等多久再抢占，默认5分钟    
    preempt delay 3
}

3.2.2 备机配置文件

文件名为keepalived_buckup.conf

! Configuration File for keepalived
global_defs {
   notification_email {
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 182.254.38.18 
   smtp_connect_timeout 30
   router_id study02
}
vrrp_script check_httpd {
    # script "/usr/bin/killall -0 httpd"
    script "/usr/bin/killall -0 nginx"
    interval 2
}
vrrp_instance VI_1 {
    state  BACKUP 
    interface eth0
    virtual_router_id 51
    priority 95
    advert_int 1
    # advert_int 0.5
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    notify_master "/etc/keepalived/master.sh" 
    notify_backup "/etc/keepalived/backup.sh"
    notify_fault "/etc/keepalived/fault.sh"
    track_script {
        check_httpd 
    }
    virtual_ipaddress {
        # 192.168.31.9 
        172.29.0.20/24 dev eth0 label eth0:0 
    }
    preempt delay 3
}

3.2.3 主备切换通知脚本

master.sh

#!/bin/bash
LOGFILE=/var/log/keepalived-state.log
:>$LOGFILE
echo "[MASTER]" >> $LOGFILE
date >> $LOGFILE

backup.sh

#!/bin/bash
LOGFILE=/var/log/keepalived-state.log
:>$LOGFILE
echo "[BACKUP]" >> $LOGFILE
date >> $LOGFILE

fault.sh

#!/bin/bash
LOGFILE=/var/log/keepalived-state.log
echo "[Fault]" >> $LOGFILE
date >> $LOGFILE

3.3 编制nginx 主页面

3.3.1 主机页面

index-master.html

    
    
    
    



    

        master
    

3.3.1 备机页面

index-slave.html

    
    
    
    




    

        slave
    

4、 验证

4.1 启动容器并查看状态

进入docker-compose 对应的目录执行

docker-compose up

进入主机shell，使用ifconfig，结果如下：

进入备机shell,结果如下：

出现如上结果说明，系统主备已经启动

也可以分别查看 keepalived-state.log 文件中的主备状态

more /var/log/keepalived-state.log 

如果出现问题可用，如下指令查看日志

tail -f /var/log/message

4.2 双机主备切换验证

4.2.1 关闭主机

关闭主机后，备机自动出现服务ip
当主机再次启动后，服务ip切换至主机

4.2.2 关闭keepalived 服务

关闭keepalived 服务后，服务ip自动切换至备机
当主机keepalived服务再次启动后，服务ip自动切换至主机

4.2.3 关闭nginx服务

停止nginx服务后，服务ip自动切换至备机
当nginx服务重新启动后，服务ip自动切换至主机