kubernetes中网络报错问题

#系统版本
cat /etc/redhat-release
CentOS Linux release 7.4.1708 (Core)

#kubelet版本
kubelet --version
Kubernetes v1.10.0

#selinux状态
getenforce
Disabled

#系统防火墙状态
systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
Active: inactive (dead)
 Docs: man:firewalld(1)

#dns的Pod 一直处于 Waiting 或 ContainerCreating 状态
kubectl get po -n kube-system
NAME                                    READY     STATUS             RESTARTS   AGE
kube-dns-86f4d74b45-ffwjf        0/3       ContainerCreating   0          6m

#查看Pod详细情况
kubectl  describe pod kube-dns-86f4d74b45-ffwjf  -n kube-system
##我们看到如下信息：
Error syncing pod
Pod sandbox changed, it will be killed and re-created.
##可以发现，该 Pod 的 Sandbox 容器无法正常启动，具体原因需要查看 Kubelet 日志。

#查看Pod的log
journalctl -u kubelet
##看到如下报错内容：
RunPodSandbox from runtime service failed: rpc error: code = 2 desc = NetworkPlugin cni failed to set up pod "kube-dns-86f4d74b45-ffwjf" network: failed to set bridge addr: "cni0" already has an IP address different from 10.244.4.1/24

##说明
这里的一个Pod中启动了多个容器，所以，我们使用kubectl logs 命令查看日志很有局限性，关于kubectl logs的使用，请参考kubernetes中的Pod简述与实践和kubernetes中文文档。

#在master节点之外的节点进行操作
kubeadm reset
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig docker0 down
ip link delete cni0
ip link delete flannel.1
##重启kubelet
systemctl restart kubelet
##重启docker
systemctl restart docker

#说明
##如果上面操作之后还是报相同的错误或是如下错误：
"CreatePodSandbox for pod \" kube-dns-86f4d74b45-ffwjf _default(78e796f5-e
b7c-11e7-b903-b827ebd42d30)\" failed: rpc error: code = Unknown desc = N
etworkPlugin cni failed to set up pod \" kube-dns-86f4d74b45-ffwjf _default\"
network: failed to allocate for range 0: no IP addresses available in range set:
10.244.1.1-10.244.1.254"

#执行如下操作步骤：
##在master主机上
kubeadm reset
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig docker0 down
ip link delete cni0
ip link delete flannel.1
##重启kubelet
systemctl restart kubelet
##重启docker
systemctl restart docker

##初始化
kubeadm init --kubernetes-version=v1.10.1 --pod-network-cidr=10.244.0.0/16
--apiserver-advertise-address=10.0.0.39
##说明：
最后给出了将节点加入集群的命令：
kubeadm join 10.0.0.39:6443 --token 4g0p8w.w5p29ukwvitim2ti 
--discovery-token-ca-cert-hash sha256:21d0adbfcb409dca97e65564
1573b2ee51c
77a212f194e20a307cb459e5f77c8
这条命令一定保存好，因为后期没法重现的！！

##建立.kube
rm -rf /root/.kube/
mkdir -p /root/.kube/
cp -i /etc/kubernetes/admin.conf /root/.kube/config
chown root:root /root/.kube/config

#在node（非master）节点上
kubeadm reset
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig docker0 down
ip link delete cni0
ip link delete flannel.1
##重启kubelet
systemctl restart kubelet
##重启docker
systemctl restart docker

## kubeadm join
kubeadm join 10.0.0.39:6443 --token 4g0p8w.w5p29ukwvitim2ti 
--discovery-token-ca-cert-hash sha256:21d0adbfcb409dca97e65564
1573b2ee51c
77a212f194e20a307cb459e5f77c8