kubernetes 部署

127.0.0.1 localhost
192.168.88.10 k8s-m1
192.168.88.11 k8s-m2
192.168.88.12 k8s-m3

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

sudo add-apt-repository    "deb [arch=amd64] https://download.docker.com/linux/ubuntu    $(lsb_release -cs)    stable"
   
sudo apt-get update

#所有节点时间同步计划任务
0 * * * * ntpdate 202.112.10.36

sudo apt-get install docker-ce=18*
sudo systemctl start docker;sudo systemctl enable docker

for node in k8s-m{1,2,3};do ssh root@$node sudo apt-get install -y kubelet=1.18* kubeadm=1.18* kubectl=1.18* ipvsadm;done
for node in k8s-m{1,2,3};do ssh root@$node sudo systemctl enable kubelet.service;done

export ETCD_version=v3.3.12
export ETCD_SSL_DIR=/etc/etcd/ssl
export APISERVER_IP=192.168.88.200
export SYSTEM_SERVICE_DIR=/usr/lib/systemd/system

for node in k8s-m{1,2,3};do ssh root@$node sudo mkdir -p /usr/lib/systemd/system ;done

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

wget https://github.com/etcd-io/etcd/releases/download/${ETCD_version}/etcd-${ETCD_version}-linux-amd64.tar.gz
tar -zxvf etcd-${ETCD_version}-linux-amd64.tar.gz && cd etcd-${ETCD_version}-linux-amd64
#复制可执行文件到其余master节点
for node in k8s-m{1,2,3};do scp etcd* root@$node:/usr/local/bin/;done

mkdir -p ${ETCD_SSL_DIR} && cd ${ETCD_SSL_DIR}
cat > ca-config.json <<EOF
{"signing":{"default":{"expiry":"87600h"},"profiles":{"kubernetes":{"usages":["signing","key encipherment","server auth","client auth"],"expiry":"87600h"}}}}
EOF
cat > etcd-ca-csr.json <<EOF 
{"CN":"etcd","key":{"algo":"rsa","size":2048},"names":[{"C":"CN","ST":"BeiJing","L":"BeiJing","O":"etcd","OU":"etcd"}]}
EOF
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=127.0.0.1,192.168.88.10,192.168.88.11,192.168.88.12 -profile=kubernetes etcd-ca-csr.json | cfssljson -bare etcd
rm -rf *.json *.csr
for node in k8s-m{2,3};do ssh root@$node mkdir -p ${ETCD_SSL_DIR} /var/lib/etcd;scp * root@$node:${ETCD_SSL_DIR};done

cat etcd_config.sh
#!/bin/bash
ETCD_URL="k8s-m1=https://192.168.88.10:2380,k8s-m2=https://192.168.88.11:2380,k8s-m3=https://192.168.88.12:2380"
IP=$(cat /etc/netplan/50-cloud-init.yaml |grep 192|head -n 1 |cut -d "-" -f 2 |cut -d "/" -f 1)
NODE=$(hostname)

cat > /etc/etcd/config <<EOF 
#[Member]
ETCD_NAME="$NODE"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://$IP:2380"
ETCD_LISTEN_CLIENT_URLS="https://$IP:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://$IP:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://$IP:2379"
ETCD_INITIAL_CLUSTER="ETCD_URL"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

cat > /usr/lib/systemd/system/etcd.service <<EOF 
[Unit]
Description=Etcd Server
After=neCNork.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/etcd/config
ExecStart=/usr/local/bin/etcd --name=${ETCD_NAME} --data-dir=${ETCD_DATA_DIR} --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} --initial-cluster=${ETCD_INITIAL_CLUSTER} --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} --initial-cluster-state=new --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --peer-cert-file=/etc/etcd/ssl/etcd.pem --peer-key-file=/etc/etcd/ssl/etcd-key.pem --trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem --peer-trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

systemctl start etcd.service;systemctl enable etcd.service

for node in k8s-m{2,3};do scp /etc/etcd/config root@$node:/etc/etcd/;scp ${SYSTEM_SERVICE_DIR}/etcd.service root@$node:${SYSTEM_SERVICE_DIR};done
for node in k8s-m{1,2,3};do scp etcd_config.sh root@$node:/tmp/ ;done
for node in k8s-m{1,2,3};do  root@$node bash /tmp/etcd_config.sh ;done

etcdctl --ca-file=/etc/etcd/ssl/etcd-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --endpoints="https://192.168.88.10:2379,https://192.168.88.11:2379,https://192.168.88.12:2379" cluster-health

for node in k8s-m{1,2,3};do ssh root@$node yum -y install haproxy keepalived;done
#检查网卡名称并修改
cat > /etc/keepalived/keepalived.conf << EOF 
vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"
    interval 3
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens192
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
    ${APISERVER_IP}
    }
     track_script {
        check_haproxy
     }
}
EOF
cat > /etc/keepalived/check_haproxy.sh <<EOF
#!/bin/bash
systemctl status haproxy > /dev/null
if [[ \$? != 0 ]];then
        echo "haproxy is down,close the keepalived"
        systemctl stop keepalived
fi
EOF
cat > /etc/haproxy/haproxy.cfg << EOF 
global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
#---------------------------------------------------------------------
frontend  k8s-api 
   bind *:8443
   mode tcp
   default_backend             apiserver
#---------------------------------------------------------------------
backend apiserver
    balance     roundrobin
    mode tcp
    server  k8s-m1 192.168.88.10:6443 check weight 1 maxconn 2000 check inter 2000 rise 2 fall 3
    server  k8s-m2 192.168.88.11:6443 check weight 1 maxconn 2000 check inter 2000 rise 2 fall 3
EOF

for node in k8s-m{2,3};do scp /etc/keepalived/* root@$node:/etc/keepalived;scp /etc/haproxy/haproxy.cfg root@$node:/etc/haproxy;done

for node in k8s-m{1,2,3};do ssh root@$node systemctl enable --now keepalived haproxy;done
#查看VIP是否工作正常ping ${APISERVER_IP} -c 3

#查看需要拉取的容器镜像
kubeadm config images list

for node in k8s-m{1,2,3};do images=(kube-apiserver:v1.18.0 kube-controller-manager:v1.18.0 kube-scheduler:v1.18.0 kube-proxy:v1.18.0 pause:3.2 coredns:1.6.7);done

  for image in ${images[@]};do docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/${image};docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/${image} k8s.gcr.io/${image};docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/${image};done

cat > kubeadm-config.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.18.0

controlPlaneEndpoint: "192.168.88.200:8443"
etcd:
  external:
    endpoints:
    - https://192.168.88.10:2379
    - https://192.168.88.11:2379
    - https://192.168.88.12:2379
    caFile: /etc/etcd/ssl/etcd-ca.pem
    certFile: /etc/etcd/ssl/etcd.pem
    keyFile: /etc/etcd/ssl/etcd-key.pem
networking:
  podSubnet: 10.244.0.0/16

kubeletConfiguration:
  baseConfig:
    evictionHard:
      imagefs.available: 2Gi
      memory.available: 512Mi
      nodefs.available: 2Gi
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF

root@node02:~/release-v3.11.2/k8s-manifests# kubectl get node 
NAME     STATUS   ROLES    AGE   VERSION
node01   Ready    master   19h   v1.18.0
node02   Ready    master   19h   v1.18.0
node03   Ready    <none>   19h   v1.18.0

root@node02:~/release-v3.11.2/k8s-manifests# kubectl get pod -o wide -n kube-system 
NAME                                       READY   STATUS    RESTARTS   AGE    IP              NODE     NOMINATED NODE   READINESS GATES
calico-kube-controllers-69d565f6ff-cf9gf   1/1     Running   0          4h1m   192.168.88.12   node03   <none>           <none>
calico-node-9hq7l                          1/1     Running   0          4h1m   192.168.88.10   node01   <none>           <none>
calico-node-hxcfv                          1/1     Running   0          4h1m   192.168.88.11   node02   <none>           <none>
calico-node-m7d6c                          1/1     Running   0          4h1m   192.168.88.12   node03   <none>           <none>
coredns-66bff467f8-g262t                   1/1     Running   0          19h    10.10.186.193   node03   <none>           <none>
coredns-66bff467f8-jf72n                   1/1     Running   0          19h    10.10.186.192   node03   <none>           <none>
kube-apiserver-node01                      1/1     Running   1          19h    192.168.88.10   node01   <none>           <none>
kube-apiserver-node02                      1/1     Running   1          19h    192.168.88.11   node02   <none>           <none>
kube-controller-manager-node01             1/1     Running   1          19h    192.168.88.10   node01   <none>           <none>
kube-controller-manager-node02             1/1     Running   1          19h    192.168.88.11   node02   <none>           <none>
kube-proxy-dl9w8                           1/1     Running   3          19h    192.168.88.12   node03   <none>           <none>
kube-proxy-rmwrp                           1/1     Running   3          19h    192.168.88.10   node01   <none>           <none>
kube-proxy-tcm74                           1/1     Running   3          19h    192.168.88.11   node02   <none>           <none>
kube-scheduler-node01                      1/1     Running   2          19h    192.168.88.10   node01   <none>           <none>
kube-scheduler-node02                      1/1     Running   2          19h    192.168.88.11   node02   <none>           <none>

ETCD_ENDPOINTS="https://192.168.88.10:2379,https://192.168.88.11:2379,https://192.168.88.12:2379"
sed -i "s#.*etcd_endpoints:.*#  etcd_endpoints: \"${ETCD_ENDPOINTS}\"#g" calico-etcd.yaml 

ETCD_CERT=`cat /etc/etcd/ssl/etcd.pem | base64 | tr -d ‘\n‘`
ETCD_KEY=`cat /etc/etcd/ssl/etcd-key.pem | base64 | tr -d ‘\n‘`
ETCD_CA=`cat /etc/etcd/ssl/etcd-ca.pem | base64 | tr -d ‘\n‘`

sed -i "s#.*etcd-cert:.*#  etcd-cert: ${ETCD_CERT}#g" calico-etcd.yaml 
sed -i "s#.*etcd-key:.*#  etcd-key: ${ETCD_KEY}#g" calico-etcd.yaml 
sed -i "s#.*etcd-ca:.*#  etcd-ca: ${ETCD_CA}#g" calico-etcd.yaml 

sed -i ‘s#.*etcd_ca:.*#  etcd_ca: "/calico-secrets/etcd-ca"#g‘ calico-etcd.yaml
sed -i ‘s#.*etcd_cert:.*#  etcd_cert: "/calico-secrets/etcd-cert"#g‘ calico-etcd.yaml
sed -i ‘s#.*etcd_key:.*#  etcd_key: "/calico-secrets/etcd-key"#g‘  calico-etcd.yaml


sed -i ‘/CALICO_IPV4POOL_CIDR/{n;s/192.168.0.0/10.10.0.0/g}‘ calico-etcd.yaml 
kubectl apply -f calico-etcd.yaml 

tunl0     Link encap:IPIP Tunnel  HWaddr   
          inet addr:10.244.0.1  Mask:255.255.255.255
          UP RUNNING NOARP  MTU:1440  Metric:1
          RX packets:6025 errors:0 dropped:0 overruns:0 frame:0
          TX packets:5633 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1 
          RX bytes:5916925 (5.9 MB)  TX bytes:1600038 (1.6 MB)

root@ub1604-k8s231:~# ip route  |grep bird
10.244.0.0/24 via 10.96.141.233 dev ens160  proto bird  #其他node配置的网络
blackhole 10.244.1.0/24  proto bird                     #本机node分配的网络
10.244.2.0/24 via 10.96.141.232 dev ens160  proto bird 
10.244.3.0/24 via 10.96.141.234 dev ens160  proto bird 
10.244.4.0/24 via 10.96.141.235 dev ens160  proto bird

kubectl create namespace test 
cat << EOF | kubectl create -f -
apiVersion: v1
kind: Pod
metadata:
  name: network-test
  namespace: test
spec:
  containers:
  - image: busybox:latest
    command:
      - sleep
      - "3600"
    name: network-test
EOF

root@node02:~/release-v3.11.2/k8s-manifests# kubectl get pod -o wide -n test 
NAME           READY   STATUS    RESTARTS   AGE     IP              NODE     NOMINATED NODE   READINESS GATES
network-test   1/1     Running   3          3h57m   10.10.186.194   node03   <none>           <none>