本文将介绍通过二进制方式部署etcd集群和api-server

部署etcd集群

生成etcd证书和私钥

#新建一个目录存放etcd的证书
cd target && mkdir etcd && cd etcd
#生成证书配置文件
cat > etcd-csr.json<<EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "10.0.40.101",
    "10.0.40.102",
    "10.0.40.103"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Hangzhou",
      "L": "Hangzhou",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

#使用ca证书生成证书、私钥
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

分发证书

scp etcd*.pem root@10.0.40.101:/etc/kubernetes/pki/
scp etcd*.pem root@10.0.40.102:/etc/kubernetes/pki/
scp etcd*.pem root@10.0.40.103:/etc/kubernetes/pki/

创建service文件

cat > etcd-40-101.service<<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/etcd-io/etcd

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/kubernetes/bin/etcd \\
  --advertise-client-urls=https://10.0.40.101:2379 \\
  --cert-file=/etc/kubernetes/pki/etcd.pem \\
  --client-cert-auth \\
  --data-dir=/var/lib/etcd \\
  --initial-advertise-peer-urls=https://10.0.40.101:2380 \\
  --initial-cluster=etcd-40-101.jieee.xyz=https://10.0.40.101:2380,etcd-40-102.jieee.xyz=https://10.0.40.102:2380,etcd-40-103.jieee.xyz=https://10.0.40.103:2380 \\
  --key-file=/etc/kubernetes/pki/etcd-key.pem \\
  --listen-client-urls=https://10.0.40.101:2379,http://127.0.0.1:2379 \\
  --listen-metrics-urls=http://10.0.40.101:2381,http://127.0.0.1:2381 \\
  --listen-peer-urls=https://10.0.40.101:2380 \\
  --name=etcd-40-101.jieee.xyz \\
  --peer-cert-file=/etc/kubernetes/pki/etcd.pem \\
  --peer-client-cert-auth \\
  --peer-key-file=/etc/kubernetes/pki/etcd-key.pem \\
  --peer-trusted-ca-file=/etc/kubernetes/pki/ca.pem \\
  --snapshot-count=10000 \\
  --trusted-ca-file=/etc/kubernetes/pki/ca.pem \\
  --initial-cluster-token=etcd-cluster-0 \\
  --initial-cluster-state=new
Restart=always
RestartSec=10s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target
EOF

注意：文件中相关的url、hostname需要修改成每个节点对应的ip

分发service文件

scp etcd-40-101.service root@10.0.40.101:/etc/systemd/system/etcd.service
scp etcd-40-102.service root@10.0.40.102:/etc/systemd/system/etcd.service
scp etcd-40-103.service root@10.0.40.103:/etc/systemd/system/etcd.service

#创建数据目录
ssh root@10.0.40.101 "mkdir -p /var/lib/etcd"
ssh root@10.0.40.102 "mkdir -p /var/lib/etcd"
ssh root@10.0.40.103 "mkdir -p /var/lib/etcd"

启动服务（在每台etcd节点上执行）

#启动服务（必须要3台同时启动，否则会启动失败）
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

#查看状态
service etcd status

#查看启动日志
journalctl -f -u etcd

部署api-server

生成api-server证书和私钥

#新建一个目录存放etcd的证书
cd target && mkdir apiserver && cd apiserver
#生成证书配置文件
cat > kubernetes-csr.json<<EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "10.0.50.101",
    "10.0.50.102",
    "10.0.50.103",
    "10.0.50.254",
    "10.120.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Hangzhou",
      "L": "Hangzhou",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

#10.0.50.254 是给apiserver高可用准备的vip，10.120.0.1是apiserver的service ip，一般是k8s service cidr的第一个ip

#使用ca证书生成证书、私钥
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

分发证书

scp kubernetes*.pem root@10.0.50.101:/etc/kubernetes/pki/
scp kubernetes*.pem root@10.0.50.102:/etc/kubernetes/pki/
scp kubernetes*.pem root@10.0.50.103:/etc/kubernetes/pki/

创建service文件

cat > kube-apiserver-50-101.service<<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/opt/kubernetes/bin/kube-apiserver \\
  --advertise-address=10.0.50.101 \\
  --allow-privileged=true \\
  --authorization-mode=Node,RBAC \\
  --client-ca-file=/etc/kubernetes/pki/ca.pem \
  --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
  --enable-bootstrap-token-auth \\
  --etcd-cafile=/etc/kubernetes/pki/ca.pem \\
  --etcd-certfile=/etc/kubernetes/pki/kubernetes.pem \\
  --etcd-keyfile=/etc/kubernetes/pki/kubernetes-key.pem \\
  --etcd-servers=https://10.0.40.101:2379,https://10.0.40.102:2379,https://10.0.40.103:2379 \\
  --insecure-port=0 \\
  --kubelet-client-certificate=/etc/kubernetes/pki/kubernetes.pem \\
  --kubelet-client-key=/etc/kubernetes/pki/kubernetes-key.pem \\
  --bind-address=0.0.0.0 \\
  --service-cluster-ip-range=10.120.0.0/16 \\
  --anonymous-auth=false \\
  --service-node-port-range=30000-31000 \\
  --tls-cert-file=/etc/kubernetes/pki/kubernetes.pem \\
  --tls-private-key-file=/etc/kubernetes/pki/kubernetes-key.pem \\
  --service-account-key-file=/etc/kubernetes/pki/ca-key.pem \\
  --enable-swagger-ui=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=30 \\
  --audit-log-maxbackup=3 \\
  --audit-log-maxsize=100 \\
  --audit-log-path=/var/log/kube-apiserver-audit.log \\
  --event-ttl=1h \\
  --alsologtostderr=true \\
  --logtostderr=false \\
  --log-dir=/var/log/kubernetes \\
  --v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

注意：文件中相关的ip需要修改成每个节点对应的ip

分发service文件

scp kube-apiserver-50-101.service root@10.0.50.101:/etc/systemd/system/kube-apiserver.service
scp kube-apiserver-50-102.service root@10.0.50.102:/etc/systemd/system/kube-apiserver.service
scp kube-apiserver-50-103.service root@10.0.50.103:/etc/systemd/system/kube-apiserver.service

#创建日志目录
ssh root@10.0.50.101 "mkdir -p /var/log/kubernetes"
ssh root@10.0.50.102 "mkdir -p /var/log/kubernetes"
ssh root@10.0.50.103 "mkdir -p /var/log/kubernetes"

启动服务

#启动服务
systemctl daemon-reload && systemctl enable kube-apiserver && systemctl start kube-apiserver

#查看运行状态
service kube-apiserver status

#查看日志
journalctl -f -u kube-apiserver

#检查监听端口
netstat -ntlp

ApiServer高可用配置（haproxy节点）

以下内容在两个节点上都要执行

安装haproxy+keepalived

#如果不加这个配置，keepalived可能会起不来
cat >> /etc/sysctl.conf<<EOF
net.ipv4.ip_nonlocal_bind = 1
EOF
sysctl -p

#安装
yum install keepalived haproxy -y

配置haproxy

vim haproxy.cfg
#====以下是配置文件内容
global
    ...
defaults
    mode   tcp   #修改成四层代理

frontend main *:6443
	default_backend    apiserver

backend apiserver
    mode        tcp 
    balance     roundrobin 
    server  master1 10.0.50.101:6443 check # 3台master节点
    server  master2 10.0.50.102:6443 check 
    server  master3 10.0.50.103:6443 check 
#====完

启动haproxy

setsebool -P haproxy_connect_any=1
systemctl enable haproxy && service haproxy start

#检查状态
service haproxy status

#访问测试
curl -k https://127.0.0.1:6443/

配置keepalived

cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived
global_defs {
  router_id apiserver
}

vrrp_script check_haproxy {
  script "/etc/keepalived/check_haproxy.sh"
  interval 3
  weight -20
}

vrrp_instance VI-apiserver {
  state MASTER    # 备节点这里配置为backup
  interface ens192    # 网卡名称
  virtual_router_id 50
  priority 100   # 备节点配置为90，必须必主节点权重小
  dont_track_primary
  advert_int 3
  virtual_ipaddress {
    10.0.50.254   # 为apiserver准备的VIP
  }
  track_script {
    check_haproxy
  }
}
EOF

#haproxy健康检查脚本
cat > /etc/keepalived/check_haproxy.sh<<EOF
#!/bin/bash
active_status=\`netstat -lntp|grep haproxy|wc -l\`
if [ \$active_status -gt 0 ]; then
    exit 0
else
    exit 1
fi
EOF

启动keepalived

systemctl enable keepalived && service keepalived start

#检查状态
service keepalived status

#查看VIP绑定情况
ip a

#访问测试
curl -k https://10.0.50.254:6443/

部分配置文件，详见GITEE