k8s开启ipvs

[root@k8s-master ~]# lsmod|grep ip_vs
ip_vs_sh 12688 0 
ip_vs_wrr 12697 0 
ip_vs_rr 12600 0 
ip_vs 141092 6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack 133387 9 ip_vs,nf_nat,nf_nat_ipv4,nf_nat_ipv6,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_netlink,nf_conntrack_ipv4,nf_conntrack_ipv6
libcrc32c 12644 4 xfs,ip_vs,nf_nat,nf_conntrack
[root@k8s-master ~]# yum install ipvsadm ipset -y

[root@k8s-master ~]# kubectl edit configmap kube-proxy -n kube-system
ipvs:
      excludeCIDRs: null
      minSyncPeriod: 0s
      scheduler: ""
      strictARP: false
      syncPeriod: 0s
      tcpFinTimeout: 0s
      tcpTimeout: 0s
      udpTimeout: 0s
    kind: KubeProxyConfiguration
    metricsBindAddress: ""
    mode: "ipvs"     #修改此处
    nodePortAddresses: null
 

[root@k8s-master ~]# kubectl get pod -n kube-system 
NAME                                 READY   STATUS    RESTARTS   AGE
coredns-6d56c8448f-bl6ds             1/1     Running   0          78m
coredns-6d56c8448f-g2scb             1/1     Running   0          78m
etcd-k8s-master                      1/1     Running   1          78m
kube-apiserver-k8s-master            1/1     Running   1          78m
kube-controller-manager-k8s-master   1/1     Running   1          78m
kube-flannel-ds-5wwvj                1/1     Running   0          76m
kube-flannel-ds-9hcqz                1/1     Running   0          77m
kube-flannel-ds-ct6jr                1/1     Running   1          76m
kube-proxy-5ntj4                     1/1     Running   0          76m
kube-proxy-82dk4                     1/1     Running   0          78m
kube-proxy-s9jrw                     1/1     Running   0          76m
kube-scheduler-k8s-master            1/1     Running   1          78m
[root@k8s-master ~]# kubectl delete pod kube-proxy-5ntj4 kube-proxy-82dk4 kube-proxy-s9jrw  -n kube-system 
pod "kube-proxy-5ntj4" deleted
pod "kube-proxy-82dk4" deleted
pod "kube-proxy-s9jrw" deleted

[root@k8s-master ~]# kubectl get pod -n kube-system 
NAME                                 READY   STATUS    RESTARTS   AGE
coredns-6d56c8448f-bl6ds             1/1     Running   0          79m
coredns-6d56c8448f-g2scb             1/1     Running   0          79m
etcd-k8s-master                      1/1     Running   1          79m
kube-apiserver-k8s-master            1/1     Running   1          79m
kube-controller-manager-k8s-master   1/1     Running   1          79m
kube-flannel-ds-5wwvj                1/1     Running   0          76m
kube-flannel-ds-9hcqz                1/1     Running   0          77m
kube-flannel-ds-ct6jr                1/1     Running   1          76m
kube-proxy-c2mxx                     1/1     Running   0          13s
kube-proxy-j4zlw                     1/1     Running   0          6s
kube-proxy-jffp7                     1/1     Running   0          16s
kube-scheduler-k8s-master            1/1     Running   1          79m
[root@k8s-master ~]# kubectl logs kube-proxy-c2mxx -n kube-system 
I0907 04:23:26.102780       1 node.go:136] Successfully retrieved node IP: 10.3.104.56
I0907 04:23:26.102846       1 server_others.go:111] kube-proxy node IP is an IPv4 address (10.3.104.56), assume IPv4 operation
I0907 04:23:26.133916       1 server_others.go:259] Using ipvs Proxier.
E0907 04:23:26.134077       1 proxier.go:381] can‘t set sysctl net/ipv4/vs/conn_reuse_mode, kernel version must be at least 4.1
W0907 04:23:26.134167       1 proxier.go:434] IPVS scheduler not specified, use rr by default
I0907 04:23:26.134396       1 server.go:650] Version: v1.19.0
I0907 04:23:26.134922       1 conntrack.go:52] Setting nf_conntrack_max to 131072
I0907 04:23:26.135295       1 config.go:224] Starting endpoint slice config controller
I0907 04:23:26.135324       1 shared_informer.go:240] Waiting for caches to sync for endpoint slice config
I0907 04:23:26.135368       1 config.go:315] Starting service config controller
I0907 04:23:26.135373       1 shared_informer.go:240] Waiting for caches to sync for service config
I0907 04:23:26.235476       1 shared_informer.go:247] Caches are synced for service config 
I0907 04:23:26.235488       1 shared_informer.go:247] Caches are synced for endpoint slice config 

[root@k8s-master ~]# kubectl get svc --all-namespaces
NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       kubernetes   ClusterIP   10.96.0.1    <none>        443/TCP                  80m
default       nginx        ClusterIP   None         <none>        80/TCP                   75m
kube-system   kube-dns     ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP,9153/TCP   80m
[root@k8s-master ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.96.0.1:443 rr
  -> 10.3.104.51:6443             Masq    1      0          0         
TCP  10.96.0.10:53 rr
  -> 10.244.0.2:53                Masq    1      0          0         
  -> 10.244.0.3:53                Masq    1      0          0         
TCP  10.96.0.10:9153 rr
  -> 10.244.0.2:9153              Masq    1      0          0         
  -> 10.244.0.3:9153              Masq    1      0          0         
UDP  10.96.0.10:53 rr
  -> 10.244.0.2:53                Masq    1      0          0         
  -> 10.244.0.3:53                Masq    1      0          0