二进制方式搭建Kubernetes 1.19.3高可用集群(二)——部署ETCD集群和api-server

发布时间:2020-11-01 21:14:48阅读:(220)

本文将介绍通过二进制方式部署etcd集群和api-server

部署etcd集群

生成etcd证书和私钥

#新建一个目录存放etcd的证书
cd target && mkdir etcd && cd etcd
#生成证书配置文件
cat > etcd-csr.json<<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"10.0.40.101",
"10.0.40.102",
"10.0.40.103"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hangzhou",
"L": "Hangzhou",
"O": "k8s",
"OU": "System"
}
]
}
EOF

#使用ca证书生成证书、私钥
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

分发证书

scp etcd*.pem root@10.0.40.101:/etc/kubernetes/pki/
scp etcd*.pem root@10.0.40.102:/etc/kubernetes/pki/
scp etcd*.pem root@10.0.40.103:/etc/kubernetes/pki/

创建service文件

cat > etcd-40-101.service<<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/etcd-io/etcd

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/kubernetes/bin/etcd \\
--advertise-client-urls=https://10.0.40.101:2379 \\
--cert-file=/etc/kubernetes/pki/etcd.pem \\
--client-cert-auth \\
--data-dir=/var/lib/etcd \\
--initial-advertise-peer-urls=https://10.0.40.101:2380 \\
--initial-cluster=etcd-40-101.jieee.xyz=https://10.0.40.101:2380,etcd-40-102.jieee.xyz=https://10.0.40.102:2380,etcd-40-103.jieee.xyz=https://10.0.40.103:2380 \\
--key-file=/etc/kubernetes/pki/etcd-key.pem \\
--listen-client-urls=https://10.0.40.101:2379,http://127.0.0.1:2379 \\
--listen-metrics-urls=http://10.0.40.101:2381,http://127.0.0.1:2381 \\
--listen-peer-urls=https://10.0.40.101:2380 \\
--name=etcd-40-101.jieee.xyz \\
--peer-cert-file=/etc/kubernetes/pki/etcd.pem \\
--peer-client-cert-auth \\
--peer-key-file=/etc/kubernetes/pki/etcd-key.pem \\
--peer-trusted-ca-file=/etc/kubernetes/pki/ca.pem \\
--snapshot-count=10000 \\
--trusted-ca-file=/etc/kubernetes/pki/ca.pem \\
--initial-cluster-token=etcd-cluster-0 \\
--initial-cluster-state=new
Restart=always
RestartSec=10s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target
EOF

注意:文件中相关的url、hostname需要修改成每个节点对应的ip

分发service文件

scp etcd-40-101.service root@10.0.40.101:/etc/systemd/system/etcd.service
scp etcd-40-102.service root@10.0.40.102:/etc/systemd/system/etcd.service
scp etcd-40-103.service root@10.0.40.103:/etc/systemd/system/etcd.service

#创建数据目录
ssh root@10.0.40.101 "mkdir -p /var/lib/etcd"
ssh root@10.0.40.102 "mkdir -p /var/lib/etcd"
ssh root@10.0.40.103 "mkdir -p /var/lib/etcd"

启动服务(在每台etcd节点上执行)

#启动服务(必须要3台同时启动,否则会启动失败)
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

#查看状态
service etcd status

#查看启动日志
journalctl -f -u etcd

部署api-server

生成api-server证书和私钥

#新建一个目录存放etcd的证书
cd target && mkdir apiserver && cd apiserver
#生成证书配置文件
cat > kubernetes-csr.json<<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"10.0.50.101",
"10.0.50.102",
"10.0.50.103",
"10.0.50.254",
"10.120.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hangzhou",
"L": "Hangzhou",
"O": "k8s",
"OU": "System"
}
]
}
EOF

#10.0.50.254 是给apiserver高可用准备的vip,10.120.0.1是apiserver的service ip,一般是k8s service cidr的第一个ip

#使用ca证书生成证书、私钥
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

分发证书

scp kubernetes*.pem root@10.0.50.101:/etc/kubernetes/pki/
scp kubernetes*.pem root@10.0.50.102:/etc/kubernetes/pki/
scp kubernetes*.pem root@10.0.50.103:/etc/kubernetes/pki/

创建service文件

cat > kube-apiserver-50-101.service<<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/opt/kubernetes/bin/kube-apiserver \\
--advertise-address=10.0.50.101 \\
--allow-privileged=true \\
--authorization-mode=Node,RBAC \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \
--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
--enable-bootstrap-token-auth \\
--etcd-cafile=/etc/kubernetes/pki/ca.pem \\
--etcd-certfile=/etc/kubernetes/pki/kubernetes.pem \\
--etcd-keyfile=/etc/kubernetes/pki/kubernetes-key.pem \\
--etcd-servers=https://10.0.40.101:2379,https://10.0.40.102:2379,https://10.0.40.103:2379 \\
--insecure-port=0 \\
--kubelet-client-certificate=/etc/kubernetes/pki/kubernetes.pem \\
--kubelet-client-key=/etc/kubernetes/pki/kubernetes-key.pem \\
--bind-address=0.0.0.0 \\
--service-cluster-ip-range=10.120.0.0/16 \\
--anonymous-auth=false \\
--service-node-port-range=30000-31000 \\
--tls-cert-file=/etc/kubernetes/pki/kubernetes.pem \\
--tls-private-key-file=/etc/kubernetes/pki/kubernetes-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/ca-key.pem \\
--enable-swagger-ui=true \\
--apiserver-count=3 \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/log/kube-apiserver-audit.log \\
--event-ttl=1h \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/var/log/kubernetes \\
--v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

注意:文件中相关的ip需要修改成每个节点对应的ip

分发service文件

scp kube-apiserver-50-101.service root@10.0.50.101:/etc/systemd/system/kube-apiserver.service
scp kube-apiserver-50-102.service root@10.0.50.102:/etc/systemd/system/kube-apiserver.service
scp kube-apiserver-50-103.service root@10.0.50.103:/etc/systemd/system/kube-apiserver.service

#创建日志目录
ssh root@10.0.50.101 "mkdir -p /var/log/kubernetes"
ssh root@10.0.50.102 "mkdir -p /var/log/kubernetes"
ssh root@10.0.50.103 "mkdir -p /var/log/kubernetes"

启动服务

#启动服务
systemctl daemon-reload && systemctl enable kube-apiserver && systemctl start kube-apiserver

#查看运行状态
service kube-apiserver status

#查看日志
journalctl -f -u kube-apiserver

#检查监听端口
netstat -ntlp

ApiServer高可用配置(haproxy节点)

以下内容在两个节点上都要执行

安装haproxy+keepalived

#如果不加这个配置,keepalived可能会起不来
cat >> /etc/sysctl.conf<<EOF
net.ipv4.ip_nonlocal_bind = 1
EOF
sysctl -p

#安装
yum install keepalived haproxy -y

配置haproxy

vim haproxy.cfg
#====以下是配置文件内容
global
...
defaults
mode tcp #修改成四层代理

frontend main *:6443
default_backend apiserver

backend apiserver
mode tcp
balance roundrobin
server master1 10.0.50.101:6443 check # 3台master节点
server master2 10.0.50.102:6443 check
server master3 10.0.50.103:6443 check
#====完

启动haproxy

setsebool -P haproxy_connect_any=1
systemctl enable haproxy && service haproxy start

#检查状态
service haproxy status

#访问测试
curl -k https://127.0.0.1:6443/

配置keepalived

cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived
global_defs {
router_id apiserver
}

vrrp_script check_haproxy {
script "/etc/keepalived/check_haproxy.sh"
interval 3
weight -20
}

vrrp_instance VI-apiserver {
state MASTER # 备节点这里配置为backup
interface ens192 # 网卡名称
virtual_router_id 50
priority 100 # 备节点配置为90,必须必主节点权重小
dont_track_primary
advert_int 3
virtual_ipaddress {
10.0.50.254 # 为apiserver准备的VIP
}
track_script {
check_haproxy
}
}
EOF

#haproxy健康检查脚本
cat > /etc/keepalived/check_haproxy.sh<<EOF
#!/bin/bash
active_status=\`netstat -lntp|grep haproxy|wc -l\`
if [ \$active_status -gt 0 ]; then
exit 0
else
exit 1
fi
EOF

启动keepalived

systemctl enable keepalived && service keepalived start

#检查状态
service keepalived status

#查看VIP绑定情况
ip a

#访问测试
curl -k https://10.0.50.254:6443/

部分配置文件,详见GITEE

标签:k8s

发表评论

评论列表(有1条评论220人围观)
游客2022-05-12 16:17:32

这文档不太行