（三）Kubernetes/K8s 高可用架構

高可用架構 采用多master+多lb+keepalive的方案實現（注意其中lb的ip 地址的證書）
1，多master安裝
將原master 1 部署文件拷貝到新master,修改bind 地址 啟動 注意 由于需要連接etcd 需要拷貝etcd 的證書

scp –r /opt/kubernetes root@192.168.0.102:/opt
scp –r /opt/etcd/ssl root@192.168.0.102:/opt/etcd
scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.31.64:/usr/lib/systemd/system
##修改apiserver配置文件為本地IP
vim  /opt/kubernetes/cfg/kube-apiserver.conf
--bind-address=192.168.0.102 \
--secure-port=6443 \
--advertise-address=192.168.0.102 \
#啟動
for i in $(ls /opt/kubernetes/bin/);do systemctl start $i;systemctl enable $i; done

lb部署
lb--nginx-mater

#安裝Nginx+Keepalived
rpm -ivh http://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.16.0-1.el7.ngx.x86_64.rpm
vim /etc/nginx/nginx.conf 
##注意添加的配置文件的位置 特別是需要inclued 插入放于events  和http 之間
stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;   ## 安裝好 之后不需要 可以關閉 日志量比較大

    upstream k8s-apiserver {
                server 192.168.0.101:6443;    ## 后端安裝master 地址
                server 192.168.0.102:6443;
            }

    server {
       listen 6443;
       proxy_pass k8s-apiserver;
    }
}

keepalived 高可用

yum install keepalived
vi /etc/keepalived/keepalived.conf
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER
} 

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
        }

vrrp_instance VI_1 { 
    state MASTER 
    interface ens33  ## 自己機器網卡的名稱
    virtual_router_id 51 # VRRP 路由 ID實例，每個實例是唯一的 
    priority 100    # 優先級，備服務器設置 90 
    advert_int 1    # 指定VRRP 心跳包通告間隔時間，默認1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.0.105/24    ## 虛擬vip地址
    } 
    track_script {
        check_nginx
    } 
}

判斷nginx 進程腳本

#!/bin/bash
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
systemctl start keepalived
systemctl enable keepalived

lb-nginx-backup
nginx 安裝同上

yum install keepalived
vi /etc/keepalived/keepalived.conf
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_BACKUP
} 

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state BACKUP      ##標注角色
    interface ens33    ##自己網卡的名稱
    virtual_router_id 51 # VRRP 路由 ID實例，每個實例是唯一的 
    priority 90    # 優先級，備服務器設置 90 
    advert_int 1    # 指定VRRP 心跳包通告間隔時間，默認1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.0.105/24
    } 
    track_script {
        check_nginx
    } 
}

# cat /etc/keepalived/check_nginx.sh 
#!/bin/bash
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi

# systemctl start keepalived
# systemctl enable keepalived

測試

##在lb master節點
ip addr show  ## 查看vip 是否綁定到了網卡上面 關閉nginx 查看vip 是否能漂移到backup 上面

修改所有node apiserver地址填寫為vip

cd /opt/kubernetes/cfg
grep 192 *
bootstrap.kubeconfig:    server: https://192.168.31.63:6443
kubelet.kubeconfig:    server: https://192.168.31.636443
kube-proxy.kubeconfig:    server: https://192.168.31.63:6443

#批量修改：
sed -i 's#192.168.0.63#192.168.0.105 *
 curl -k --header "Authorization: Bearer c47ffb939f5ca36231d9e3121a252940" https://192.168.31.60:6443/version
##此teken 是bootstarp 的token