前言：

接上一篇多节点部署（2）部署负载均衡

部署环境

负载均衡
Nginx1:192.168.13.128/24
Nginx2:192.168.13.129/24
Master节点
master1:192.168.13.131/24 kube-apiserver kube-controller-manager kube-scheduler etcd
master2:192.168.13.130/24 kube-apiserver kube-controller-manager kube-scheduler etcd
Node节点
node1:192.168.13.132/24 kubelet kube-proxy docker flannel etcd
node2:192.168.13.133/24 kubelet kube-proxy docker flannel etcd

负载均衡部署

1，在两台Nginx服务器上安装nginx服务

[root@nginx01 ~]# rz -E    ##上传nginx脚本和keepalive配置文件
[root@nginx01 ~]# ls      
keepalived.conf   nginx.sh 
[root@nginx01 ~]# systemctl stop firewalld.service   ##关闭防火墙
[root@nginx01 ~]# setenforce 0

vim nginx.sh   ##nginx脚本
cat > /etc/yum.repos.d/nginx.repo << EOF
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
EOF

stream {

     log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
        access_log  /var/log/nginx/k8s-access.log  main;

        upstream k8s-apiserver {
                server 10.0.0.3:6443;
                server 10.0.0.8:6443;
        }
        server {
                                listen 6443;
                                proxy_pass k8s-apiserver;
        }
        }

[root@nginx01 ~]# vim /etc/yum.repos.d/nginx.repo   ##配置nginx的yum源
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
[root@nginx01 ~]# yum list   ##更新yum
[root@nginx01 ~]# yum install -y nginx  ##下载Nginx

2，修改两台nginx服务器的配置文件

[root@nginx01 ~]# vim /etc/nginx/nginx.conf
events {
        worker_connections  1024;
}     ##在此处下面添加四层转发配置

stream {

     log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent'; 
        access_log  /var/log/nginx/k8s-access.log  main;

        upstream k8s-apiserver {
                server 192.168.13.131:6443;   ##master01地址
                server 192.168.13.130:6443;   ##master02地址
        }   
        server {
                                listen 6443;
                                proxy_pass k8s-apiserver;
        }           
        }
[root@nginx01 ~]# systemctl start nginx  ##开启nginx服务
##可以修改/usr/share/nginx/html/index.html主页区分主master从backup
##浏览器查看两个nginx网站

3，在两台nginx上部署keepalived服务，并修改配置文件

[root@nginx01 ~]# yum install -y keepalived  ##安装keepalived服务
[root@nginx01 ~]# cp keepalived.conf /etc/keepalived/keepalived.conf  ##复制配置文件
[root@nginx01 ~]# vim /etc/keepalived/keepalived.conf  ##主master的配置文件修改
! Configuration File for keepalived 

global_defs { 
     # 接收邮件地址 
     notification_email { 
         acassen@firewall.loc 
         failover@firewall.loc 
         sysadmin@firewall.loc 
     } 
     # 邮件发送地址 
     notification_email_from Alexandre.Cassen@firewall.loc  
     smtp_server 127.0.0.1 
     smtp_connect_timeout 30 
     router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
        script "/etc/nginx/check_nginx.sh"     ##nginx检查脚本，需要自己去编辑的
}

vrrp_instance VI_1 { 
        state MASTER    ##主服务
        interface ens33
        virtual_router_id 51 ## VRRP 路由 ID实例，每个实例是唯一的 
        priority 100    ## 优先级，备服务器设置 90 
        advert_int 1    ## 指定VRRP 心跳包通告间隔时间，默认1秒 
        authentication { 
                auth_type PASS      ##验证不需要修改，主从一致
                auth_pass 1111 
        }  
        virtual_ipaddress { 
                192.168.13.100/24     ##虚拟ip地址
        } 
        track_script {
                check_nginx
        } 
}

[root@nginx02 ~]# vim /etc/keepalived/keepalived.conf  ##备backup的配置文件修改
! Configuration File for keepalived 

global_defs { 
     # 接收邮件地址 
     notification_email { 
         acassen@firewall.loc 
         failover@firewall.loc 
         sysadmin@firewall.loc 
     } 
     # 邮件发送地址 
     notification_email_from Alexandre.Cassen@firewall.loc  
     smtp_server 127.0.0.1 
     smtp_connect_timeout 30 
     router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
        script "/etc/nginx/check_nginx.sh"     ##nginx检查脚本
}

vrrp_instance VI_1 { 
        state BACKUP 
        interface ens33
        virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
        priority 90    # 优先级，备服务器设置 90 
        advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
        authentication { 
                auth_type PASS      
                auth_pass 1111 
        }  
        virtual_ipaddress { 
                192.168.13.100/24 
        } 
        track_script {
                check_nginx
        } 
}
[root@nginx01 ~]#  vim /etc/nginx/check_nginx.sh   ##编辑nginx检查脚本
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
        systemctl stop keepalived
fi
[root@nginx01 ~]# chmod +x /etc/nginx/check_nginx.sh   ##给执行权限
[root@nginx01 ~]# systemctl start keepalived.service   ##开启服务
[root@nginx01 ~]# ip a   ##查看地址信息
        inet 192.168.13.100/24 scope global secondary ens33   ##漂移地址在master上

4，验证地址漂移

##在nginx01中关闭nginx服务，此时keepalived服务也关闭了（check_nginx.sh）
[root@nginx01 ~]# pkill nginx
##在nginx02中查看漂移地址
[root@nginx02 ~]# ip a  ##此时13.100在nginx02上
##在nginx01上恢复nginx和keepalived服务，查看漂移地址
[root@nginx01 ~]# systemctl start nginx
[root@nginx01 ~]# systemctl start keepalived.service 
[root@nginx01 ~]# ip a   ##此时漂移地址又到了nginx01上
##用浏览器访问虚拟ip

5，在所有的node节点服务器上统一配置VIP

[root@node01 ~]# vim /opt/kubernetes/cfg/bootstrap.kubeconfig 
server: https://192.168.13.100:6443
[root@node01 ~]# vim /opt/kubernetes/cfg/kubelet.kubeconfig
server: https://192.168.13.100:6443
[root@node01 ~]# vim /opt/kubernetes/cfg/kube-proxy.kubeconfig
server: https://192.168.13.100:6443
[root@node01 ~]# cd /opt/kubernetes/cfg/  ##切换到配置文件目录
[root@node01 cfg]# grep 100 *   ##查看修改的情况
bootstrap.kubeconfig:    server: https://192.168.13.100:6443
kubelet.kubeconfig:    server: https://192.168.13.100:6443
kube-proxy.kubeconfig:    server: https://192.168.13.100:6443
[root@node01 cfg]# systemctl restart kubelet.service   ##重启两个服务
[root@node01 cfg]# systemctl restart kube-proxy.service
##在nginx01上查看访问日志
[root@nginx01 ~]# tail /var/log/nginx/k8s-access.log
192.168.13.132 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.132 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.132 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.133 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.133 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0

6，在matser01上创建pod

[root@master01 ~]# kubectl get pods   ##查看pod
No resources found.
[root@master01 ~]# kubectl run nginx --image=nginx   ##创建pod
[root@master01 ~]# kubectl get pods    ##查看pod状态是正在创建的状态
NAME                    READY   STATUS              RESTARTS   AGE
nginx-dbddb74b8-brjlj   0/1     ContainerCreating   0          86s
[root@master01 ~]# kubectl get pods   ##此时pod是已经运行的状态
NAME                    READY   STATUS    RESTARTS   AGE
nginx-dbddb74b8-brjlj   1/1     Running   0          87s
[root@master01 ~]# kubectl logs nginx-dbddb74b8-brjlj  ##此时日志文件不能查看
[root@master01 ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
##提权后日志文件就可以查看了
[root@master01 ~]# kubectl get pods -o wide  ##查看pod网络，此时pod容器分配到node01上
NAME                    READY   STATUS    RESTARTS   AGE     IP            NODE             NOMINATED NODE
nginx-dbddb74b8-brjlj   1/1     Running   0          5m18s   172.17.45.2   192.168.13.132   <none>

7，在对应的网段node01上访问

[root@node01 cfg]# curl 172.17.45.2  ##此时就可以访问nginx信息
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
##用node01虚拟机的浏览器访问

8，此时在master01上查看日志文件

[root@master01 ~]# kubectl logs nginx-dbddb74b8-brjlj                             
172.17.45.1 - - [10/Feb/2020:05:29:23 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"

负载均衡部署完成，未完待续...