Docker安装

方式一：yum源安装

# 安装依赖包
yum install -y yum-utils device-mapper-persistent-data lvm2

# 添加Docker软件包源
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

#关闭测试版本list（只显示稳定版）
sudo yum-config-manager --disable docker-ce-edge
sudo yum-config-manager --disable docker-ce-test

#更新yum包索引
yum makecache fast

#NO.1 直接安装Docker CE （will always install the highest  possible version，可能不符合你的需求）
yum install docker-ce

#NO.2 指定版本安装
yum list docker-ce --showduplicates|sort -r  
yum install -y docker-ce-18.09.5 docker-ce-cli-18.09.5

方式二：rpm包安装

# 官网下载rpm包
https://download.docker.com/linux/centos/7/x86_64/stable/Packages

#上传服务器，安装
yum install  /path/安装包.rpm
# 启动
systemctl start docker
# 测试
docker run hello-world
docker version
#升级（下载新版本包）
yum upgrade  /path/新版本.rpm

方式三：官方提供一键安装脚本（不推荐生产环境使用！）

用于开发环境快速安装，不提供版本选择，总是安装最新的edg版和test版
脚本需要root权限执行
不要在已经用其他方式安装docker 的服务器上使用脚本安装

get.docker.com 总是安装Docker CE 的edge最新版本；（edg版，每月更新的版本，新特性，bug多）
# curl -fsSL get.docker.com -o get-docker.sh
# sudo sh get-docker.sh

test.docker.com 总是安装Docker CE 的test最新版本
# curl -fsSL test.docker.com -o test-docker.sh
# sudo sh test-docker.sh

Kubernetes镜像安装

# 安装docker-ce

# 修改docker cgroup驱动：native.cgroupdriver=systemd
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
EOF

# 配置kubernetes源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 关闭SElinux
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

# 安装kubelet kubeadm kubectl
yum install -y kubelet-1.14.1 kubeadm-1.14.1 kubectl-1.14.1 --disableexcludes=kubernetes

systemctl enable --now kubelet  # 开机启动kubelet

# centos7用户还需要设置路由
yum install -y bridge-utils.x86_64
modprobe  br_netfilter  # 加载br_netfilter模块，使用lsmod查看开启的模块
cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

# 重新加载所有配置文件
sysctl --system  

# 关闭防火墙
systemctl disable --now firewalld  

# k8s要求关闭swap  (qxl)
swapoff -a && sysctl -w vm.swappiness=0  # 关闭swap
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab  # 取消开机挂载swap

创建集群准备工作

# Master端：
kubeadm config images pull # 拉取集群所需镜像，这个需要翻墙

# --- 不能翻墙可以尝试以下办法 ---
kubeadm config images list # 列出所需镜像
#(不是一定是下面的,根据实际情况来)
# 根据所需镜像名字先拉取国内资源
kubeadm config images pull

docker pull mirrorgooglecontainers/kube-apiserver:v1.14.1
docker pull mirrorgooglecontainers/kube-controller-manager:v1.14.1
docker pull mirrorgooglecontainers/kube-scheduler:v1.14.1
docker pull mirrorgooglecontainers/kube-proxy:v1.14.1
docker pull mirrorgooglecontainers/pause:3.1
docker pull mirrorgooglecontainers/etcd:3.3.10
docker pull coredns/coredns:1.3.1

# 修改镜像tag
docker tag mirrorgooglecontainers/kube-apiserver:v1.14.1 k8s.gcr.io/kube-apiserver:v1.14.1
docker tag mirrorgooglecontainers/kube-controller-manager:v1.14.1 k8s.gcr.io/kube-controller-manager:v1.14.1
docker tag mirrorgooglecontainers/kube-scheduler:v1.14.1 k8s.gcr.io/kube-scheduler:v1.14.1
docker tag mirrorgooglecontainers/kube-proxy:v1.14.1 k8s.gcr.io/kube-proxy:v1.14.1
docker tag mirrorgooglecontainers/pause:3.1 k8s.gcr.io/pause:3.1
docker tag mirrorgooglecontainers/etcd:3.3.10 k8s.gcr.io/etcd:3.3.10
docker tag coredns/coredns:1.3.1 k8s.gcr.io/coredns:1.3.1

# 删除原来的镜像
docker rmi mirrorgooglecontainers/kube-apiserver:v1.14.1
docker rmi mirrorgooglecontainers/kube-controller-manager:v1.14.1
docker rmi mirrorgooglecontainers/kube-scheduler:v1.14.1
docker rmi mirrorgooglecontainers/kube-proxy:v1.14.1
docker rmi mirrorgooglecontainers/pause:3.1
docker rmi mirrorgooglecontainers/etcd:3.3.10
docker rmi coredns/coredns:1.3.1

# --- 不能翻墙可以尝试使用 ---
# Node端：
# 根据所需镜像名字先拉取国内资源
docker pull mirrorgooglecontainers/kube-proxy:v1.14.1
docker pull mirrorgooglecontainers/pause:3.1

# 修改镜像tag
docker tag mirrorgooglecontainers/kube-proxy:v1.14.1 k8s.gcr.io/kube-proxy:v1.14.1
docker tag mirrorgooglecontainers/pause:3.1 k8s.gcr.io/pause:3.1

# 删除原来的镜像
docker rmi mirrorgooglecontainers/kube-proxy:v1.14.1
docker rmi mirrorgooglecontainers/pause:3.1

# 获取网络插件flannel
docker pull yqfwind/flannel:v0.10.0-amd64
docker tag yqfwind/flannel:v0.10.0-amd64 quay.io/coreos/flannel:v0.10.0-amd64
docker rmi yqfwind/flannel:v0.10.0-amd64
或
docker pull registry.cn-beijing.aliyuncs.com/imcto/flannel:v0.11.0-amd64
docker tag registry.cn-beijing.aliyuncs.com/imcto/flannel:v0.11.0-amd64 quay.io/coreos/flannel:v0.11.0-amd64
docker rmi registry.cn-beijing.aliyuncs.com/imcto/flannel:v0.11.0-amd64

K8S master节点初始化与启动

# 使用kubeadm创建集群
# 初始化Master（Master需要至少2核）此处会各种报错,异常...成功与否就在此
kubeadm init --apiserver-advertise-address 192.168.190.101 --kubernetes-version=v1.14.1 --pod-network-cidr 10.244.0.0/16
# --apiserver-advertise-address 指定与其它节点通信的接口
# --pod-network-cidr 指定pod网络子网，使用fannel网络必须使用这个CIDR。10.244.0.0/16为后面下载的kube-flannel.yml文件中设定的，必须保持一致
# 运行初始化，程序会检验环境一致性，可以根据实际错误提示进一步修复问题。
程序会访问https://dl.k8s.io/release/stable-1.txt获取最新的k8s版本，访问这个连接需要FQ，如果无法访问，则会使用kubeadm client的版本作为安装的版本号，使用kubeadm version查看client版本。也可以使用--kubernetes-version明确指定版本。

# kubeadm reset可在master和minion节点上重置，以便再次执行kubeadm init和kubeadm join
[root@centos76-101 ~]# kubeadm init --apiserver-advertise-address 192.168.190.101 --kubernetes-version=v1.14.1 --pod-network-cidr 10.244.0.0/16
I0811 17:03:15.235074   50695 version.go:96] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get https://dl.k8s.io/release/stable-1.txt: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
I0811 17:03:15.235304   50695 version.go:97] falling back to the local client version: v1.14.1
[init] Using Kubernetes version: v1.14.1
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [centos76-101 localhost] and IPs [192.168.190.101 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [centos76-101 localhost] and IPs [192.168.190.101 127.0.0.1 ::1]
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [centos76-101 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.190.101]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 21.507377 seconds
[upload-config] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.14" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --experimental-upload-certs
[mark-control-plane] Marking the node centos76-101 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node centos76-101 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: 3bamfh.zlwg81jysrr3axah
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.190.101:6443 --token 3bamfh.zlwg81jysrr3axah \
    --discovery-token-ca-cert-hash sha256:44667289d8b83ed9908c09faf5f766965aacf581edf798e50dfc7b2c468d088a

# 普通用户设置权限
rm -rf $HOME/.kube
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

# 保存下面这段代码，后面在minion节点上执行，以让这些节点加入集群
kubeadm join 192.168.190.101:6443 --token 3bamfh.zlwg81jysrr3axah \
    --discovery-token-ca-cert-hash sha256:44667289d8b83ed9908c09faf5f766965aacf581edf798e50dfc7b2c468d088a

运行flannel网络插件

# 下载资源描述文件kube-flannel.yml
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

# 启动网络插件（/xxx/kube-flannel.yml替换为刚才下载的kube-flannel.yml文件路径）
kubectl apply -f /xxx/kube-flannel.yml

# 自此master节点上的pod全部启动正常
[root@centos76-101 ~]# kubectl get pods -n kube-system -o wide
NAME                                   READY   STATUS    RESTARTS   AGE   IP                NODE           NOMINATED NODE   READINESS GATES
coredns-fb8b8dccf-c49pl                1/1     Running   0          64s   10.244.0.7        centos76-101   <none>           <none>
coredns-fb8b8dccf-dggpb                1/1     Running   0          64s   10.244.0.10       centos76-101   <none>           <none>
etcd-centos76-101                      1/1     Running   0          26s   192.168.190.101   centos76-101   <none>           <none>
kube-apiserver-centos76-101            1/1     Running   0          3s    192.168.190.101   centos76-101   <none>           <none>
kube-controller-manager-centos76-101   1/1     Running   0          22s   192.168.190.101   centos76-101   <none>           <none>
kube-flannel-ds-amd64-9qg6g            1/1     Running   0          43s   192.168.190.101   centos76-101   <none>           <none>
kube-proxy-s44vm                       1/1     Running   0          64s   192.168.190.101   centos76-101   <none>           <none>

minion节点加入集群

# minion节点上执行
kubeadm join 192.168.190.101:6443 --token 3bamfh.zlwg81jysrr3axah \
    --discovery-token-ca-cert-hash sha256:44667289d8b83ed9908c09faf5f766965aacf581edf798e50dfc7b2c468d088a

# master节点上执行，可见所有k8s pod在集群各节点上运行正常
[root@centos76-101 ~]# kubectl get pods -n kube-system -o wide
NAME                                   READY   STATUS    RESTARTS   AGE    IP                NODE           NOMINATED NODE   READINESS GATES
coredns-fb8b8dccf-c49pl                1/1     Running   0          119s   10.244.0.7        centos76-101   <none>           <none>
coredns-fb8b8dccf-dggpb                1/1     Running   0          119s   10.244.0.10       centos76-101   <none>           <none>
etcd-centos76-101                      1/1     Running   0          81s    192.168.190.101   centos76-101   <none>           <none>
kube-apiserver-centos76-101            1/1     Running   0          58s    192.168.190.101   centos76-101   <none>           <none>
kube-controller-manager-centos76-101   1/1     Running   0          77s    192.168.190.101   centos76-101   <none>           <none>
kube-flannel-ds-amd64-46xbc            1/1     Running   0          38s    192.168.190.103   centos76-103   <none>           <none>
kube-flannel-ds-amd64-9qg6g            1/1     Running   0          98s    192.168.190.101   centos76-101   <none>           <none>
kube-flannel-ds-amd64-cf8vv            1/1     Running   0          44s    192.168.190.102   centos76-102   <none>           <none>
kube-proxy-pz5dn                       1/1     Running   0          44s    192.168.190.102   centos76-102   <none>           <none>
kube-proxy-qwn2f                       1/1     Running   0          38s    192.168.190.103   centos76-103   <none>           <none>
kube-proxy-s44vm                       1/1     Running   0          119s   192.168.190.101   centos76-101   <none>           <none>
kube-scheduler-centos76-101            1/1     Running   0          54s    192.168.190.101   centos76-101   <none>           <none>

# 节点状态都正常
[root@centos76-101 ~]# kubectl get node
NAME           STATUS   ROLES    AGE   VERSION
centos76-101   Ready    master   32m   v1.14.1
centos76-102   Ready    <none>   30m   v1.14.1
centos76-103   Ready    <none>   30m   v1.14.1

问题记录

# 问题1：coredns启动后为CrashLoopBackOff状态，日志显示pod为路由错误
root@centos76-101 ~]# kubectl logs coredns-fb8b8dccf-km5n5 -n kube-system 
E0404 15:59:25.549577       1 reflector.go:134] github.com/coredns/coredns/plugin/kubernetes/controller.go:315: Failed to list *v1.Service: Get https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0: dial tcp 10.96.0.1:443: connect: no route to host
E0404 15:59:25.549577       1 reflector.go:134] github.com/coredns/coredns/plugin/kubernetes/controller.go:315: Failed to list *v1.Service: Get https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0: dial tcp 10.96.0.1:443: connect: no route to host
log: exiting because of error: log: cannot create log: open /tmp/coredns.coredns-fb8b8dccf-km5n5.unknownuser.log.ERROR.20200404-155925.1: no such file or directory

# 解决方法，参考[kubedns container cannot connect to apiserver](https://github.com/kubernetes/kubeadm/issues/193)

systemctl stop kubelet
systemctl stop docker
iptables --flush
iptables -tnat --flush
systemctl start kubelet
systemctl start docker