Kubernetes二进制安装

架构图

实验架构图

准备工作

准备虚拟机

5台服务器

操作系统

centos7.6 1810 mini安装

调整yum源

安装epel-release

# yum install -y epel-relase

关闭SElinux和firewalld

# sed -i '/SELINUX/{s/permissive/disabled/}' /etc/selinux/config

# setenforce 0 #查看状态(getenforce)

# systemctl stop firewalld

安装必要工具

# yum install -y wget net-tools telnet tree nmap sysstat lrzsz dos2unix bind-utils

DNS服务初始化

pl66-240.host.com上：

安装bind9软件

# yum install -y bind

配置bind9

主配置文件

# vim /etc/named.conf

listen-on port 53 { 10.10.66.240; };
~~listen-on-v6 port 53 { ::1; };~~
allow-query     { localhost; }; >>allow-query     { any; }; 
forwarders      { 8.8.8.8; };
dnssec-enable no;
dnssec-validation no;

检查配置，是否配置格式正确。

# named-checkconf

区域配置文件

# vim /etc/named.rfc1912.zones

zone "host.com" IN {
type master;
file "host.com.zone";
allow-update { 10.10.66.240; };
};
zone "yw.com" IN {
type master;
file "yw.com.zone";
allow-update { 10.10.66.240; };
};

配置区域数据文件

配置主机域数据文件

# vim /var/named/host.com.zone

$ORIGIN host.com.
$TTL 600        ; 10 minutes
@   IN SOA dns.host.com. dnsadmin.host.com. (
                20200306    ; serial
                10800       ; refresh (3 hours)
                900     ; retry (15 minutes)
                604800      ; expire (1 week)
                86400       ; minimum (1 day)
                )
            NS  dns.host.com.
$TTL 60 ; 1 minute
dns     A   10.10.66.240
pl66-240        A   10.10.66.240
pl66-241        A   10.10.66.241
pl66-242        A   10.10.66.242
pl66-243        A   10.10.66.243
pl66-245        A   10.10.66.245

# vim /var/named/yw.com.zone

$ORIGIN yw.com.
$TTL 600        ; 10 minutes
@   IN SOA dns.yw.com. dnsadmin.yw.com. (
                20200306    ; serial
                10800       ; refresh (3 hours)
                900     ; retry (15 minutes)
                604800      ; expire (1 week)
                86400       ; minimum (1 day)
                )
            NS  dns.yw.com.
$TTL 60 ; 1 minute
dns     A   10.10.66.240

检查配置，是否配置格式正确。

# named-checkconf

启动bind9

# named-checkconf
# systemctl start named
# systemctl enable named

检查

[root@pl66-240 ~]# dig -t A pl66-242.host.com @10.10.66.240 +short
10.10.66.242
[root@pl66-240 ~]# dig -t A dns.yw.com @10.10.66.240 +short
10.10.66.240

批量修改其他主机dns

[root@pl66-240 ~]# cat /etc/resolv.conf
search host.com
nameserver 10.10.66.240

将240主机上的resolve文件拷贝到其他主机

# ansible server -m copy -a 'src=/etc/resolv.conf dest=/etc/resolv.conf force=yes backup=yes'

准备签发证书环境

运维主机pl66-245.host.com上

安装CFSSL

证书签发工具CFSSL:R1.2
cfssl下载地址
 cfssl-json下载地址
 cfssl-certinfo下载地址

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64  -O /usr/bin/123/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/bin/123/cfssl-json
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/bin/123/cfssl-certinfo
chmod +x /usr/bin/cfssl*

创建生产CA证书的JSON 配置文件

vim /opt/certs/ca-config.json

{
    "signing": {
        "default": {
            "expiry": "175200h"
    },
        "profiles": {
            "server": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
                    "client": {
                                "expiry": "175200h",
                                "usages": [
                                        "signing",
                                        "key encipherment",
                                        "client auth"
                                ]
            },
                        "peer": {
                                "expiry": "175200h",
                                "usages": [
                                        "signing",
                                        "key encipherment",
                                        "server auth",
                    "client auth"
                                ]
                        }
        }
    }
}

证书类型

client certificate：客户端使用，用于服务端认证客户端,例如etcdctl、etcd proxy、fleetctl、docker客户端
server certificate: 服务端使用，客户端以此验证服务端身份,例如docker服务端、kube-apiserver
peer certificate: 双向证书，用于etcd集群成员间通信

创建生产CA证书签名请求（csr）的JSON配置文件

# /opt/cets/ca-csr.json

{
    "CN": "pinnet",
    "hosts": [ 
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "chengdu",
            "L": "chengdu",
            "O": "pl",
            "OU": "ops"
        }
    ],
    "ca": {
        "expiry": "175200h"
    }
}

CN: Common Name,浏览器使用该字段验证网站是否合法，一般写的是域名。
C: Country，国家
ST：State，州，省
L: Locality，地区，城市
O： Organization Name，组织名称，公司名称
OU：Organization Unit Name，组织单位名称，公司部门

生产CA证书和私钥

[root@66-245 certs]# cfssl gencert -initca ca-csr.json | cfssl-json -bare ca

输出结果如下：

[root@66-245 certs]#  cfssl gencert -initca ca-csr.json | cfssl-json -bare ca
2020/03/07 05:58:05 [INFO] generating a new CA key and certificate from CSR
2020/03/07 05:58:05 [INFO] generate received request
2020/03/07 05:58:05 [INFO] received CSR
2020/03/07 05:58:05 [INFO] generating key: rsa-2048
2020/03/07 05:58:06 [INFO] encoded CSR
2020/03/07 05:58:06 [INFO] signed certificate with serial number 64696289091365665227482443074556056282272288290
[root@66-245 certs]# ll
total 16
-rw-r--r-- 1 root root  989 Mar  7 05:58 ca.csr
-rwxr-xr-x 1 root root  224 Mar  7 05:56 ca-csr.json
-rw------- 1 root root 1675 Mar  7 05:58 ca-key.pem
-rw-r--r-- 1 root root 1338 Mar  7 05:58 ca.pem
[root@66-245 certs]#

部署docker环境

安装

$\color{blue}{pl66-242，pl66-243， pl66-245上：}$

# curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun

配置

创建docker配置文件

# vim /etc/docker/daemon.json

{
    "graph": "/data/docker",
    "storage-driver": "overlay2",
    "insecure-registries": ["registry.access.redhat.com","quay.io","harbor.yw.com"],
    "registry-mirrors": ["https://q2gr04ke.mirror.aliyuncs.com"],
    "bip": "172.16.242.1/24",
    "exec-opts": ["native.cgroupdriver=systemd"],
    "live-restore": true
}

启动脚本

/usr/lib/systemd/system/docker.service


[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
#TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process

[Install]
WantedBy=multi-user.target

启动

# systemctl start docker
# systemctl enable docker
# docker version

部署docker镜像私有仓库harbor

$\color{blue}{ pl66-245上：}$

harbor官方github地址

harbor下载地址

下载离线安装版本1.9.3

github 版本信息

解压缩包

[root@66-245 opt]# tar -xcf harbor-offline-installer-v1.9.3.tgz

为版本添加版本号

[root@66-245 opt]# mv harbor/ harbor1.9.3

软连接

[root@66-245 opt]# ln -s /opt/harbor1.9.3/ /opt//harbor
以便软件升级

harbor参数修改

hostname: harbor.yw.com
http:
  port: 180
harbor_admin_password: Harbor12345    ##生产环境需更改
data_volume: /data/harbor
log:
  level: info
  rotate_count: 50
  rotate_size: 200M
  location: /data/harbor/logs

创建harbor所需的目录

# mkdir -p /data/harbor
# mkdir -p /data/harbor/logs

安装docker-compose

$\color{Green}{harbor依赖于docker-compose做单机编排}$

[root@66-245 harbor]# yum install docker-compose -y

执行harbor安装文件

[root@66-245 harbor]# /opt/harbor/install.sh

harbor

安装完成后检查

[root@66-245 harbor]# docker-compose ps

docker-compose ps

安装nginx做反向代理

[root@66-245 harbor]# yum -y install nginx

添加Nginx反向代理配置

vim /etc/nginx/conf.d/harbor.yw.com.conf
server {
    listen  80;
    server_name harbor.yw.com;
    
    client_mx_body_size 1000m;
    
    location / {
        proxy_pass http://127.0.0.1:180;
    }   
}

在DNS服务器上（pl66-240）上添加harbor的A记录，并重启服务让其生效

dns

systemctl restart named

验证：

dig -t A harbor.yw.com +short
10.10.66.240

检查添加的配置文件

image.png

检查命令

image.png

验证harbor

打开网页，输入 http:harbor.yw.com

shu

输入账号密码登录

用户名：admin
密码：Harbor12345

新建项目

image.png

下载nginx镜像

[root@pl66-245 nginx]# docker pull nginx

查看nginx镜像ID

[root@pl66-245 nginx]# docker images

为镜像添加TAG

[root@pl66-245 nginx]# docker tag 6678c7c2e56c harbor.yw.com/public/nginx:1.9.1

将本地nginx镜像上传至harbor中

登录harbor

docker login harbor.yw.com

[root@pl66-245 nginx]# docker login harbor.yw.com
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded
[root@pl66-245 nginx]#

2.将镜像上传至harbor

docker push harbor.yw.com/public/nginx:1.9.1

[root@pl66-245 nginx]# docker push harbor.yw.com/public/nginx:1.9.1
The push refers to repository [harbor.yw.com/public/nginx]
55a77731ed26: Pushed 
71f2244bc14d: Pushed 
f2cb0ecef392: Pushed 
1.9.1: digest: sha256:3936fb3946790d711a68c58be93628e43cbca72439079e16d154b5db216b58da size: 948
[root@pl66-245 nginx]#

安装Master节点服务

部署etcd集群

集群规划

主机名	角色	IP
pl66-241	etcd lead	10.10.66.241
pl66-242	etcd follow	10.10.66.242
pl66-243	etcd follow	10.10.66.243

注意:这里部署文档以pl66-241为例，另外两台主机部署方法类似

创建生成证书签名请求（csr）的JSON配置文件

vi /opt/certs/cat etcd-peer-csr.json

{
    "CN": "k8s-etcd",
    "hosts": [ 
        "10.10.66.240",
        "10.10.66.241",
        "10.10.66.242",
        "10.10.66.243"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "chengdu",
            "L": "chengdu",
            "O": "pl",
            "OU": "ops"
        }
    ]
}

生成etcd证书和私钥

[root@pl66-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json | cfssl-json -bare etcd-peer

[root@pl66-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-c
2020/03/10 02:26:28 [INFO] generate received request
2020/03/10 02:26:28 [INFO] received CSR
2020/03/10 02:26:28 [INFO] generating key: rsa-2048
2020/03/10 02:26:28 [INFO] encoded CSR
2020/03/10 02:26:28 [INFO] signed certificate with serial number 643611486410713894911975662668229763052251494279
2020/03/10 02:26:28 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

检查生成的证书、私钥

[root@pl66-245 certs]# ll
total 36
-rw-r--r-- 1 root root  918 Mar  9 10:26 ca-config.json
-rw-r--r-- 1 root root  989 Mar  7 05:58 ca.csr
-rwxr-xr-x 1 root root  224 Mar  7 05:56 ca-csr.json
-rw------- 1 root root 1675 Mar  7 05:58 ca-key.pem
-rw-r--r-- 1 root root 1338 Mar  7 05:58 ca.pem
-rw-r--r-- 1 root root 1062 Mar 10 02:26 etcd-peer.csr
-rw-r--r-- 1 root root  262 Mar 10 01:36 etcd-peer-csr.json
-rw------- 1 root root 1675 Mar 10 02:26 etcd-peer-key.pem
-rw-r--r-- 1 root root 1424 Mar 10 02:26 etcd-peer.pem
[root@pl66-245 certs]#

创建etcd用户

在pl66-241上：

useradd -s /sbin/nologin -M etcd

[root@pl66-241 ~]# useradd -s /sbin/nologin -M etcd
[root@pl66-241 ~]# id etcd
uid=1004(etcd) gid=1004(etcd) groups=1004(etcd)

下载软件，解压，做软连接

etcd下载地址
$\color{blue}{pl66-241上：}$

[root@pl66-241 opt]# tar xvf etcd-v3.1.20-linux-amd64.tar.gz -C /opt
[root@pl66-242 opt]# mv etcd-v3.1.20-linux-amd64 etcd-v3.1.20
[root@pl66-241 opt]# ln -s /opt/etcd-v3.1.20 /opt/etcd

创建目录,拷贝证书,私钥

$\color{blue}{pl66-241上：}$

[root@pl66-241 opt]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server

拷贝证书
***将运维主机上生成的ca.pem、etcd-peer-key.pem、etcd-peer.pem拷贝到/opt/etcd/certs目录中，注意私钥
/opt/etcd/certs

[root@pl66-241 certs]# scp pl66-245:/opt/certs/ca.pem .
[root@pl66-241 certs]# scp pl66-245:/opt/certs/etcd-peer-key.pem .
[root@pl66-241 certs]# scp pl66-245:/opt/certs/etcd-peer.pem .
文件权限600***

修改权限

[root@pl66-241 certs]# ll
total 12
-rw-r--r--. 1 root root 1338 Mar 10 11:19 ca.pem
-rw-------. 1 root root 1675 Mar 10 11:20 etcd-peer-key.pem
-rw-r--r--. 1 root root 1424 Mar 10 11:20 etcd-peer.pem
[root@pl66-241 certs]#

创建etcd服务启动脚本

$\color{blue}{pl66-241上：}$

vi /opt/etcd/etcd-server-startup.sh

#!/bin/sh
./etcd --name etcd-server-66-241 \
    --data-dir /data/etcd/etcd-server \
    --listen-peer-urls https://10.10.66.241:2380 \
    --listen-client-urls https://10.10.66.241:2379,http://127.0.0.1:2379 \
    --quota-backend-bytes 800000000 \
    --initial-advertise-peer-urls https://10.10.66.241:2380 \
    --advertise-client-urls https://10.10.66.241:2379,http://127.0.0.1:2379 \
    --initial-cluster etcd-server-66-241=https://10.10.66.241:2380,etcd-server-66-242=https://10.10.66.242:2380,etcd-server-66-243=https://10.10.66.243:2380 \
    --ca-file ./certs/ca.pem \
    --cert-file ./certs/etcd-peer.pem \
    --key-file ./certs/etcd-peer-key.pem \
    --client-cert-auth \
    --trusted-ca-file ./certs/ca.pem \
    --peer-ca-file ./certs/ca.pem \
    --peer-cert-file ./certs/etcd-peer.pem \
    --peer-key-file ./certs/etcd-peer-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file ./certs/ca.pem \
    --log-output stdout

调整权限和目录

$\color{blue}{pl66-241上：}$

[root@pl66-241 etcd]# chmod +x /opt/etcd/etcd-server-startup.sh
[root@pl66-241 etcd]# chown -R etcd.etcd /opt/etcd-v3.1.20/
[root@pl66-241 etcd]# chown -R etcd.etcd /data/etcd/
[root@pl66-241 etcd]# chown -R etcd.etcd /data/logs/etcd-server/
[root@pl66-241 etcd]# mkdir -p /data/logs/etcd-server

安装supervisor软件

$\color{blue}{pl66-241上：}$
管理后台进程，如果进程挂了可以自动拉起来

[root@pl66-241 etcd]# yum -y install supervisor
[root@pl66-241 etcd]# systemctl start supervisord
[root@pl66-241 etcd]# systemctl enable supervisord

创建etcd-server的启动配置

$\color{blue}{pl66-241上：}$

/etc/suppervisord.d/etcd-server.ini

[program:etcd-server-66-241]
command=/opt/etcd/etcd-server-startup.sh
numprocs=1
directory=/opt/etcd
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=etcd
redirect_stderr=true
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

注意：etcd集群各主机启动配置略有不同，配置其他节点时注意修改。

启动etcd服务并检查

[root@pl66-241 supervisord.d]# supervisorctl update
etcd-server-66-241: added process group

查看启动日志

[root@pl66-241 supervisord.d]# tail -fn 200 /data/logs/etcd-server/etcd.stdout.log

状态检查

[root@pl66-241 member]# supervisorctl status
etcd-server-66-241               RUNNING   pid 28649, uptime 0:00:36
[root@pl66-241 member]# netstat -ntlp | grep etcd
tcp        0      0 10.10.66.241:2379       0.0.0.0:*               LISTEN      28650/./etcd        
tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      28650/./etcd        
tcp        0      0 10.10.66.241:2380       0.0.0.0:*               LISTEN      28650/./etcd

配置另外两台服务器

检查cluster状态

[root@pl66-241 etcd]# ./etcdctl cluster-health
member f8d0e74dd98768e is healthy: got healthy result from http://127.0.0.1:2379
member 53fdb991bce71f1c is healthy: got healthy result from http://127.0.0.1:2379
member 690d0b927b2d3fb7 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@pl66-241 etcd]#

[root@pl66-241 etcd]# ./etcdctl member list
f8d0e74dd98768e: name=etcd-server-66-242 peerURLs=https://10.10.66.242:2380 clientURLs=http://127.0.0.1:2379,https://10.10.66.242:2379 isLeader=false
53fdb991bce71f1c: name=etcd-server-66-243 peerURLs=https://10.10.66.243:2380 clientURLs=http://127.0.0.1:2379,https://10.10.66.243:2379 isLeader=false
690d0b927b2d3fb7: name=etcd-server-66-241 peerURLs=https://10.10.66.241:2380 clientURLs=http://127.0.0.1:2379,https://10.10.66.241:2379 isLeader=true
[root@pl66-241 etcd]#

部署kube-apiserver集群

集群规划

主机名	角色	IP
pl66-240	kube-apiserver	10.10.66.242
pl66-241	kube-apiserver	10.10.66.243
pl66-242	4层负载均衡	10.10.66.240
pl66-243	4层负载均衡	10.10.66.241

注意：这里10.10.66.240和10.10.66.241使用nginx做4层负载均衡器，用keepalived跑一个vip：10.10.66.250，代理两个kube-apiserver，实现高可用

这里部署文档以pl66-243主机为例，另外一台运算节点安装部署方法类似

下载软件，解压，创建软连接

$\color{blue}{pl66-242上：}$
kubernetes官方Github地址
 kubernetes下载地址

[root@pl66-242 opt]# tar xvf kubernetes-server-linux-arm64.tar.gz
[root@pl66-243 opt]# mv kubernetes kubernetes-v1.15.2
[root@pl66-243 opt]# ln -s kubernetes-v1.15.2 kubernetes

[root@pl66-243 opt]# ll
total 442992
drwx--x--x. 4 root root        28 Mar  7 14:43 containerd
lrwxrwxrwx. 1 root root        17 Mar 10 16:46 etcd -> /opt/etcd-v3.1.20
drwxr-xr-x. 4 etcd etcd       166 Mar 10 16:48 etcd-v3.1.20
-rw-r--r--. 1 root root   9850227 Mar 10 16:29 etcd-v3.1.20-linux-amd64.tar.gz
lrwxrwxrwx. 1 root root        18 Mar 13 10:44 kubernetes -> kubernetes-v1.15.2
-rw-r--r--. 1 root root 443770238 Mar 13 09:29 kubernetes-server-linux-amd64.tar.gz
drwxr-xr-x. 4 root root        79 Aug  5  2019 kubernetes-v1.15.2

签发client证书

$\color{blue}{pl66-245上：}$
创建生成证书签名请求（csr）的JSON配置文件

/opt/certs/client-csr.json

{
    "CN": "k8s-node",
    "hosts": [ 
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "chengdu",
            "L": "chengdu",
            "O": "pl",
            "OU": "ops"
        }
    ]
}

生成client证书和私钥

[root@pl66-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssl-json -bare client

检查生成的证书、私钥

[root@pl66-245 certs]# ls -l | grep client
-rw-r--r-- 1 root root  993 Mar 11 07:58 client.csr
-rw-r--r-- 1 root root  192 Mar 11 07:58 client-csr.json
-rw------- 1 root root 1675 Mar 11 07:58 client-key.pem
-rw-r--r-- 1 root root 1359 Mar 11 07:58 client.pem
[root@pl66-245 certs]#

签发kube-apiserver证书

$\color{blue}{pl66-245上：}$

创建生成证书签名请求（csr）的JSON配置文件

/opt/certs/apiserver-csr.json

{
    "CN": "k8s-apiserver",
    "hosts": [ 
        "127.0.0.1",
        "192.168.0.1",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.cluster",
        "kubernetes.default.cluster.local",
        "10.10.66.241",
        "10.10.66.242",
        "10.10.66.243",
        "10.10.66.250"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "chengdu",
            "L": "chengdu",
            "O": "pl",
            "OU": "ops"
        }
    ]
}

生成kube-apiserver证书和私钥

[root@pl66-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json | cfssl-json -bare apiserver

检查生成的证书、私钥

[root@pl66-245 certs]# ls -l | grep apiserver
-rw-r--r-- 1 root root 1240 Mar 11 08:47 apiserver.csr
-rw-r--r-- 1 root root  421 Mar 11 06:38 apiserver-csr.json
-rw------- 1 root root 1679 Mar 11 08:47 apiserver-key.pem
-rw-r--r-- 1 root root 1582 Mar 11 08:47 apiserver.pem
[root@pl66-245 certs]#

拷贝证书至各运算节点，并创建配置

$\color{blue}{pl66-242上：}$

拷贝证书、私钥，注意私钥文件属性600

/opt/kubernetes/server/bin/cert

[root@pl66-242 cert]# scp pl66-245:/opt/certs/ca.pem .
The authenticity of host 'pl66-245 (10.10.66.245)' can't be established.
ECDSA key fingerprint is SHA256:2YOuINoiCs2y07VJzw8hwpc4pbPES7BNYU1c01zdoBg.
ECDSA key fingerprint is MD5:63:11:13:4d:18:eb:fa:2c:9e:21:73:43:5a:51:e9:5e.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'pl66-245,10.10.66.245' (ECDSA) to the list of known hosts.
root@pl66-245's password: 
ca.pem                                                                          100% 1338   796.4KB/s   00:00    
[root@pl66-242 cert]# scp pl66-245:/opt/certs/apiserver-key.pem .
root@pl66-245's password: 
apiserver-key.pem                                                               100% 1679   998.9KB/s   00:00    
[root@pl66-242 cert]# scp pl66-245:/opt/certs/apiserver.pem .
root@pl66-245's password: 
apiserver.pem                                                                   100% 1582     1.0MB/s   00:00    
[root@pl66-242 cert]# scp pl66-245:/opt/certs/ca-key.pem .
root@pl66-245's password: 
ca-key.pem                                                                      100% 1675   913.5KB/s   00:00    
[root@pl66-242 cert]# 
[root@pl66-242 cert]# 
[root@pl66-242 cert]# scp pl66-245:/opt/certs/client-key.pem .
root@pl66-245's password: 
client-key.pem                                     n                             100% 1675   848.3KB/s   00:00    
[root@pl66-242 cert]# scp pl66-245:/opt/certs/client.pem .
root@pl66-245's password: 
client.pem                                                                      100% 1359   773.5KB/s   00:00    
[root@pl66-242 cert]# ls -l 
total 24
-rw-------. 1 root root 1679 Mar 11 08:53 apiserver-key.pem
-rw-r--r--. 1 root root 1582 Mar 11 08:53 apiserver.pem
-rw-------. 1 root root 1675 Mar 11 08:53 ca-key.pem
-rw-r--r--. 1 root root 1338 Mar 11 08:53 ca.pem
-rw-------. 1 root root 1675 Mar 11 08:54 client-key.pem
-rw-r--r--. 1 root root 1359 Mar 11 08:54 client.pem
[root@pl66-242 cert]#

创建配置

/opt/kubernetes/server/bin/conf/audit.yaml

apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
  - "RequestReceived"
rules:
  # Log pod changes at RequestResponse level
  - level: RequestResponse
    resources:
    - group: ""
      # Resource "pods" doesn't match requests to any subresource of pods,
      # which is consistent with the RBAC policy.
      resources: ["pods"]
  # Log "pods/log", "pods/status" at Metadata level
  - level: Metadata
    resources:
    - group: ""
      resources: ["pods/log", "pods/status"]

  # Don't log requests to a configmap called "controller-leader"
  - level: None
    resources:
    - group: ""
      resources: ["configmaps"]
      resourceNames: ["controller-leader"]

  # Don't log watch requests by the "system:kube-proxy" on endpoints or services
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
    - group: "" # core API group
      resources: ["endpoints", "services"]

  # Don't log authenticated requests to certain non-resource URL paths.
  - level: None
    userGroups: ["system:authenticated"]
    nonResourceURLs:
    - "/api*" # Wildcard matching.
    - "/version"

  # Log the request body of configmap changes in kube-system.
  - level: Request
    resources:
    - group: "" # core API group
      resources: ["configmaps"]
    # This rule only applies to resources in the "kube-system" namespace.
    # The empty string "" can be used to select non-namespaced resources.
    namespaces: ["kube-system"]

  # Log configmap and secret changes in all other namespaces at the Metadata level.
  - level: Metadata
    resources:
    - group: "" # core API group
      resources: ["secrets", "configmaps"]

  # Log all other resources in core and extensions at the Request level.
  - level: Request
    resources:
    - group: "" # core API group
    - group: "extensions" # Version of group should NOT be included.

  # A catch-all rule to log all other requests at the Metadata level.
  - level: Metadata
    # Long-running requests like watches that fall under this rule will not
    # generate an audit event in RequestReceived.
    omitStages:
      - "RequestReceived"

创建启动脚本

$\color{blue}{pl66-242上：}$

vi /opt/kubernetes/server/bin/kube-apiserver.sh

#!/bin/bash
./kube-apiserver \
        --apiserver-count 2 \
        --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
        --audit-policy-file ./conf/audit.yaml \
        --authorization-mode RBAC \
        --client-ca-file ./cert/ca.pem \
        --requestheader-client-ca-file ./cert/ca.pem \
        --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
        --etcd-cafile ./cert/ca.pem \
        --etcd-certfile ./cert/client.pem \
        --etcd-keyfile ./cert/client-key.pem \
        --etcd-servers https://10.10.66.241:2379,https://10.10.66.242:2379,https://10.10.66.243:2379 \
        --service-account-key-file ./cert/ca-key.pem \
        --service-cluster-ip-range 192.168.0.0/16 \
        --service-node-port-range 3000-29999 \
        --target-ram-mb=1024 \
        --kubelet-client-certificate ./cert/client.pem \
        --kubelet-client-key ./cert/client-key.pem \
        --log-dir /data/logs/kubernetes/kube-apiserver \
        --tls-cert-file ./cert/apiserver.pem \
        --tls-private-key-file ./cert/apiserver-key.pem \
        --v 2

调整权限和目录

$\color{blue}{pl66-242上：}$

chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh
mkdir -p /data/logs/kubernetes/kube-apiserver

创建supervisor配置

$\color{blue}{pl66-242上：}$

vi /etc/supervisord.d/kube-apiserver.ini

[program:kube-apiserver-66-242]
command=/opt/kubernetes/server/bin/kube-apiserver.sh
numprocs=1
directory=/opt/kubernetes/server/bin/
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

启动服务并检查

[root@pl66-242 supervisord.d]# supervisorctl update

[root@pl66-242 supervisord.d]# supervisorctl status

[root@pl66-242 supervisord.d]# supervisorctl status
etcd-server-66-242               RUNNING   pid 10902, uptime 1:49:11
kube-apiserver-66-242            RUNNING   pid 10901, uptime 1:49:11
[root@pl66-242 supervisord.d]#

$\color{blue}{pl66-243的配置与242一模一样，只有supervisor上面的ini配置参数需要改一下名字：}$

配置4层反向代理

$\color{blue}{pl66-240，pl66-241上：}$

安装nginx

[root@pl66-240 ~]# yum -y install nginx

Nginx配置

/etc/nginx/nginx.conf

stream {
    upstream kube-apiserver {
        server 10.10.66.242:6443    max_fails=3 fail_timeout=30s;
        server 10.10.66.243:6443    max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}

注意:需要在最后追加

keepalived配置

安装keepalived

[root@pl66-241 etcd]# yum -y install keepalived

创建脚本check_port.sh

vim /etc/keepalived/check_port.sh
chmod +x keepalived.conf

#!/bin/bash
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then 
    PORT_PROCESS=`ss -lnt | grep $CHK_PORT | wc -l`
    if [ $PORT_PROCESS -eq 0 ];then
        echo "Port $CHK_PORT is Not Used,End."
        exit 1
    fi
else
    echo "Check Port Cant Be Empty!"
fi

keepalive主

$\color{blue}{pl66-240上：}$

yum -y install keepalived
[root@pl66-240 keepalived]# rpm -qa keepalived
keepalived-1.3.5-16.el7.x86_64

vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
    router_id 10.10.66.240

}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state MASTER
    interface enp2s0
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.10.66.240
    nopreemt

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
        10.10.66.250
    }
}

keepalive备

$\color{blue}{pl66-241上：}$

yum -y install keepalived
[root@pl66-241 keepalived]# rpm -qa keepalived
keepalived-1.3.5-16.el7.x86_64
vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
    router_id 10.10.66.241

}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface enp3s0
    virtual_router_id 251
    mcast_src_ip 10.10.66.241
    priority 90
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
        10.10.66.250
    }
}

启动代理并检查

$\color{blue}{pl66-241 , pl66-242上：}$

systemctl start keepalived
systemctl enable keepalived
nginx -s reload

[root@pl66-240 keepalived]# ip add 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether f4:4d:30:14:4d:75 brd ff:ff:ff:ff:ff:ff
    inet 10.10.66.240/24 brd 10.10.66.255 scope global noprefixroute enp2s0
       valid_lft forever preferred_lft forever
    inet 10.10.66.250/32 scope global enp2s0
       valid_lft forever preferred_lft forever
    inet6 fe80::f64d:30ff:fe14:4d75/64 scope link 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:35:68:1c:de brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
[root@pl66-240 keepalived]#

部署controller-manager

集群规划

主机名	角色	IP
pl66-242	controller-manager	10.10.66.242
pl66-243	controller-manager	10.10.66.243

注意：这里部署文档以pl66-242主机为例，另外一台运算节点安装部署方法类似

创建启动脚本

$\color{blue}{pl66-242：}$

/opt/kubernetes/server/bin/kube-controller-manager.sh

#!/bin/sh
./kube-controller-manager \
    --cluster-cidr  172.16.0.0/16 \
    --leader-elect true \
    --log-dir /data/kubernetes/kube-controller-manager \
    --master http://127.0.0.1:8080 \
    --service-account-private-key-file ./cert/ca-key.pem \
    --service-cluster-ip-range 192.168.0.0/16 \
    --root-ca-file ./cert/ca.pem \
    --v 2

调整文件权限，创建目录

$\color{blue}{pl66-242：}$

chmod+x /opt/kubernetes/server/bin/kube-controller-manager.sh
mkdir -p /data/kubernetes/kube-controller-manager
mkdir -p /data/logs/kubernetes/kube-controller-manager

创建supervisor配置

/etc/supervisord.d/kube-controller-manager.ini

[program:kube-controller-manager--66-242]
command=/opt/kubernetes/server/bin/kube-controller-manager.sh
numprocs=1
directory=/opt/kubernetes/server/bin/
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controller.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

启动服务并检查

$\color{blue}{pl66-242：}$

[root@pl66-242 supervisord.d]# supervisorctl update

[root@pl66-242 supervisord.d]# supervisorctl status
etcd-server-66-242                RUNNING   pid 10902, uptime 8:05:27
kube-apiserver-66-242             RUNNING   pid 10901, uptime 8:05:27
kube-controller-manager--66-242   RUNNING   pid 11558, uptime 0:06:13

部署kube-scheduler

集群规划

主机名	角色	IP
pl66-242	kube-scheduler	10.10.66.242
pl66-243	kube-scheduler	10.10.66.243

注意：这里部署文档以pl66-242主机为例，另外一台运算节点安装部署方法类似

创建启动脚本

$\color{blue}{pl66-242：}$

vim /opt/kubernetes/server/bin/kube-scheduler.sh

#!/bin/bash
./kube-scheduler \
    --leader-elect \
    --log-dir /data/logs/kubernetes/kube-scheduler \
    --master http://127.0.0.1:8080 \
    --v 2

调整文件权限，创建目录

$\color{blue}{pl66-242：}$

chmod +x /opt/kubernetes/server/bin/kube-scheduler.sh
mkdir -p /data/logs/kubernetes/kube-scheduler

创建supervisor配置

vim /etc/supervisord.d/kube-scheduler.ini

[program:kube-scheduler--66-242]
command=/opt/kubernetes/server/bin/kube-scheduler.sh
numprocs=1
directory=/opt/kubernetes/server/bin/
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

添加信息并检查

[root@pl66-242 supervisord.d]# supervisorctl update

[root@pl66-242 supervisord.d]# supervisorctl status
etcd-server-66-242                RUNNING   pid 11782, uptime 0:01:23
kube-apiserver-66-242             RUNNING   pid 11762, uptime 0:01:23
kube-controller-manager--66-242   RUNNING   pid 11807, uptime 0:00:31
kube-scheduler--66-242            RUNNING   pid 11763, uptime 0:01:23
[root@pl66-242 supervisord.d]#

创建kubectl软连接

[root@pl66-242 supervisord.d]# ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl

[root@pl66-242 supervisord.d]# which kubectl
/usr/bin/kubectl

检查集群状态

[root@pl66-242 supervisord.d]# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok                   
controller-manager   Healthy   ok                   
etcd-0               Healthy   {"health": "true"}   
etcd-1               Healthy   {"health": "true"}   
etcd-2               Healthy   {"health": "true"}   
[root@pl66-242 supervisord.d]#

部署Node节点服务

部署kubelet

集群规划

主机名	角色	IP
pl66-242.host.com	kubelet	10.10.66.242
pl66-243.host.com	kubelet	10.10.66.243

注意：这里部署文档以pl66-242主机为例，另外一台运算节点安装部署方法类似

签发kubelet证书

$\color{blue}{pl66-245上：}$

创建生成证书签名请求（csr）的Json配置文件

vi /opt/certs/kubelet-csr.json

{
    "CN": "k8s-kubelet",
    "hosts": [ 
        "127.0.0.1",
        "10.10.66.240",
        "10.10.66.241",
        "10.10.66.242",
        "10.10.66.243",
        "10.10.66.250",
                "10.10.66.251",
                "10.10.66.252",
                "10.10.66.253"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "chengdu",
            "L": "chengdu",
            "O": "pl",
            "OU": "ops"
        }
    ]
}

生成kubelet证书和私钥

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssl-json -bare kubelet

[root@pl66-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssl-json -bare kubelet2020/03/16 02:18:20 [INFO] generate received request
2020/03/16 02:18:20 [INFO] received CSR
2020/03/16 02:18:20 [INFO] generating key: rsa-2048
2020/03/16 02:18:20 [INFO] encoded CSR
2020/03/16 02:18:20 [INFO] signed certificate with serial number 411291623634880987451147311712722127071427596871
2020/03/16 02:18:20 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

检查生成的证书、私钥

[root@pl66-245 certs]# ll /opt/certs/ | grep kubelet
-rw-r--r-- 1 root root 1106 Mar 16 02:18 kubelet.csr
-rw-r--r-- 1 root root  394 Mar 16 02:15 kubelet-csr.json
-rw------- 1 root root 1679 Mar 16 02:18 kubelet-key.pem
-rw-r--r-- 1 root root 1456 Mar 16 02:18 kubelet.pem
[root@pl66-245 certs]#

拷贝证书至各运算节点,并创建配置

$\color{blue}{pl66-242上：}$

拷贝证书、私钥，注意私钥文件属性600

/opt/kubernetes/server/bin/cert
[root@pl66-242 cert]# scp pl66-245:/opt/certs/kubelet-key.pem .
[root@pl66-242 cert]# scp pl66-245:/opt/certs/kubelet.pem .

[root@pl66-242 cert]# ll
total 32
-rw-------. 1 root root 1679 Mar 12 09:53 apiserver-key.pem
-rw-r--r--. 1 root root 1582 Mar 12 09:53 apiserver.pem
-rw-------. 1 root root 1675 Mar 12 09:53 ca-key.pem
-rw-r--r--. 1 root root 1338 Mar 12 09:53 ca.pem
-rw-------. 1 root root 1675 Mar 12 09:53 client-key.pem
-rw-r--r--. 1 root root 1359 Mar 12 09:53 client.pem
-rw-------. 1 root root 1679 Mar 16 02:25 kubelet-key.pem
-rw-r--r--. 1 root root 1456 Mar 16 02:25 kubelet.pem
[root@pl66-242 cert]#

创建配置

set-cluster
注意：在conf目录下

/opt/kubernetes/server/bin/conf
set-cluster

[root@pl66-242 conf]# kubectl config set-cluster myk8s \
 --certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \
 --embed-certs=true \
 --server=https://10.10.66.250:7443 \
 --kubeconfig=kubelet.kubeconfig

Cluster "myk8s" set.
[root@pl66-242 conf]#

set-credentials

[root@pl66-242 conf]# kubectl config set-credentials k8s-node \
> --client-certificate=/opt/kubernetes/server/bin/cert/client.pem \
> --client-key=/opt/kubernetes/server/bin/cert/client-key.pem \
> --embed-certs=true \
> --kubeconfig=kubelet.kubeconfig
User "k8s-node" set.

set-context

[root@pl66-242 conf]# kubectl config set-context myk8s-context \
> --cluster=myk8s \
> --user=k8s-node \
> --kubeconfig=kubelet.kubeconfig
Context "myk8s-context" created.
[root@pl66-242 conf]#

use-context

[root@pl66-242 conf]# kubectl config use-context myk8s-context --kubeconfig=kubelet.kubeconfig
Switched to context "myk8s-context".
[root@pl66-242 conf]#

k8s-node.yaml

创建资源配置文件

/opt/kubernetes/server/bin/conf/k8s-node.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: k8s-node
roleRef: 
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: k8s-node

应用资源配置文件

[root@pl66-242 conf]# kubectl create -f k8s-node.yaml 
clusterrolebinding.rbac.authorization.k8s.io/k8s-node created

检查

[root@pl66-242 cert]# kubectl get clusterrolebinding k8s-node
NAME       AGE
k8s-node   18m

拷贝kubelet配置文件、证书到243主机上

[root@pl66-242 conf]# ll
total 20
-rw-r--r--. 1 root root 2223 Mar 13 01:14 audit.yaml
-rw-r--r--. 1 root root 1003 Mar 12 10:14 audit.yaml.bak
-rw-r--r--. 1 root root  259 Mar 16 06:03 k8s-node.yaml
-rw-------. 1 root root 6178 Mar 16 06:00 kubelet.kubeconfig
[root@pl66-242 conf]# pwd
/opt/kubernetes/server/bin/conf
[root@pl66-242 conf]# scp kubelet.kubeconfig pl66-243:/opt/kubernetes/server/bin/conf/
root@pl66-243's password: 
kubelet.kubeconfig                                                                                                 100% 6178     2.3MB/s   00:00    
[root@pl66-242 conf]#

$\color{blue}{pl66-245上：}$

[root@pl66-245 certs]# scp kubelet.pem root@pl66-243:/opt/kubernetes/server/bin/cert
The authenticity of host 'pl66-243 (10.10.66.243)' can't be established.
ECDSA key fingerprint is SHA256:yghdzfvB+QjjAsNSdGlAOhu1cm2yEIVLRidqi2k3+QQ.
ECDSA key fingerprint is MD5:52:2b:f4:1b:d0:83:00:dd:62:b6:66:d2:9f:38:77:8b.
Are you sure you want to continue connecting (yes/no)? yes            
Warning: Permanently added 'pl66-243' (ECDSA) to the list of known hosts.
root@pl66-243's password: 
kubelet.pem                                                                                                        100% 1456   890.8KB/s   00:00    
[root@pl66-245 certs]# scp kubelet-key.pem root@pl66-243:/opt/kubernetes/server/bin/cert
root@pl66-243's password: 
kubelet-key.pem                                                                                                    100% 1679   442.0KB/s   00:00    
[root@pl66-245 certs]#

准备pause基础镜像

$\color{blue}{pl66-245上：}$

下载

[root@pl66-245 certs]# docker pull kubernetes/pause
Using default tag: latest
latest: Pulling from kubernetes/pause
4f4fb700ef54: Pull complete 
b9c8ec465f6b: Pull complete 
Digest: sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105
Status: Downloaded newer image for kubernetes/pause:latest
docker.io/kubernetes/pause:latest
[root@pl66-245 certs]#

提交至私有仓库(harbor)中
登录harbor

[root@pl66-245 certs]# docker login harbor.yw.com

给镜像打tag

[root@pl66-245 certs]# docker tag f9d5de079539 harbor.yw.com/public/pause:latest

push到harbor

[root@pl66-245 certs]# docker push harbor.yw.com/public/pause:latest

创建kubelet启动脚本

$\color{blue}{pl66-242上：}$

/opt/kubernetes/server/bin/kubelet.sh

#!/bin/bash

./kubelet \
    --anonymous-auth=false \
    --cgroup-driver systemd \
    --cluster-dns 192.168.0.2 \
    --cluster-domain cluster.local \
    --runtime-cgroups=/systemd/system.slice \
    --kubelet-cgroups=/systemd/system.slice \
    --fail-swap-on="false" \
    --client-ca-file ./cert/ca.pem \
    --tls-cert-file ./cert/kubelet.pem \
    --tls-private-key-file ./cert/kubelet-key.pem \
    --hostname-override pl66-242.host.com \
    --image-gc-high-threshold 20 \
    --image-gc-low-threshold 10 \
    --kubeconfig ./conf/kubelet.kubeconfig \
    --log-dir /data/logs/kubernetes/kube-kubelet \
    --pod-infra-container-image harbor.yw.com/public/pause:latest \
    --root-dir /data/kubelet

检查配置，权限，创建日志目录

$\color{blue}{pl66-242上：}$

[root@pl66-242 conf]# ll | grep kubelet
-rw-------. 1 root root 6178 Mar 16 06:00 kubelet.kubeconfig

chmod +x /opt/kubernetes/server/bin/kubelet.sh
mkdir -p /data/logs/kubernetes/kube-kubelet /data/kubelet

创建supervisor配置

$\color{blue}{pl66-242上：}$

添加角色

[root@pl66-242 ~]# kubectl label node pl66-242.host.com node-role.kubernetes.io/master=
[root@pl66-242 ~]# kubectl label node pl66-242.host.com node-role.kubernetes.io/node=

部署kube-proxy

集群规划

主机名	角色	IP
pl66-242.host.com	kube-proxy	10.10.66.242
pl66-243.host.com	kube-proxy	10.10.66.243

$\color{blue}{pl66-243的配置与242一模一样，只有supervisor上面的ini配置参数需要改一下名字：}$

创建生成证书签名请求（csr）的JSON配置文件

/opt/certs/kube-proxy-csr.json

{
    "CN": "system:kube-proxy",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "chengdu",
            "L": "chengdu",
            "O": "pl",
            "OU": "ops"
        }
    ]
}

生成证书、私钥

[root@pl66-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssl-json -bare kube-proxy-client 
2020/03/16 08:58:51 [INFO] generate received request
2020/03/16 08:58:51 [INFO] received CSR
2020/03/16 08:58:51 [INFO] generating key: rsa-2048
2020/03/16 08:58:51 [INFO] encoded CSR
2020/03/16 08:58:51 [INFO] signed certificate with serial number 601899238979766833696791320168818948790769415904
2020/03/16 08:58:51 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

检查

[root@pl66-245 certs]# ll | grep kube-proxy
-rw-r--r-- 1 root root 1005 Mar 16 08:58 kube-proxy-client.csr
-rw------- 1 root root 1675 Mar 16 08:58 kube-proxy-client-key.pem
-rw-r--r-- 1 root root 1371 Mar 16 08:58 kube-proxy-client.pem
-rw-r--r-- 1 root root  184 Mar 16 08:53 kube-proxy-csr.json
[root@pl66-245 certs]#

拷贝证书,私钥,注意私钥文件属性600

/opt/kubernetes/server/bin/cert

[root@pl66-242 cert]# scp pl66-245:/opt/cert/
apiserver-key.pem  apiserver.pem      ca-key.pem         ca.pem             client-key.pem     client.pem         
[root@pl66-242 cert]# scp pl66-245:/opt/certs/kube-proxy-client.pem .
root@pl66-245's password: 
kube-proxy-client.pem                                                                                                    100% 1371   737.8KB/s   00:00    
[root@pl66-242 cert]# scp pl66-245:/opt/certs/kube-proxy-client-key.pem .
root@pl66-245's password: 
kube-proxy-client-key.pem                                                                                                100% 1675   899.0KB/s   00:00    
[root@pl66-242 cert]# ll
total 40
-rw-------. 1 root root 1679 Mar 12 09:53 apiserver-key.pem
-rw-r--r--. 1 root root 1582 Mar 12 09:53 apiserver.pem
-rw-------. 1 root root 1675 Mar 12 09:53 ca-key.pem
-rw-r--r--. 1 root root 1338 Mar 12 09:53 ca.pem
-rw-------. 1 root root 1675 Mar 12 09:53 client-key.pem
-rw-r--r--. 1 root root 1359 Mar 12 09:53 client.pem
-rw-------. 1 root root 1679 Mar 16 02:25 kubelet-key.pem
-rw-r--r--. 1 root root 1456 Mar 16 02:25 kubelet.pem
-rw-------. 1 root root 1675 Mar 16 09:40 kube-proxy-client-key.pem
-rw-r--r--. 1 root root 1371 Mar 16 09:40 kube-proxy-client.pem
[root@pl66-242 cert]#

创建配置

set-cluster

[root@pl66-242 conf]# kubectl config set-cluster myk8s \
 --certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \
 --embed-certs=true \
 --server=https://10.10.66.250:7443 \
 --kubeconfig=kube-proxy.kubeconfig

Cluster "myk8s" set.
[root@pl66-242 conf]#

set-credentials

[root@pl66-242 conf]# kubectl config set-credentials kube-proxy \
> --client-certificate=/opt/kubernetes/server/bin/cert/kube-proxy-client.pem \
> --client-key=/opt/kubernetes/server/bin/cert/kube-proxy-client-key.pem \
> --embed-certs=true \
> --kubeconfig=kube-proxy.kubeconfig

User "k8s-node" set.

set-context

[root@pl66-242 conf]# kubectl config set-context myk8s-context \
> --cluster=myk8s \
> --user=k8s-proxy \
> --kubeconfig=kube-proxy.kubeconfig

Context "myk8s-context" created.

[root@pl66-242 conf]#

use-context

[root@pl66-242 conf]# kubectl config use-context myk8s-context --kubeconfig=kube-proxy.kubeconfig
Switched to context "myk8s-context".

拷贝配置文件到PL66-243上。

[root@pl66-242 conf]# ll
total 28
-rw-r--r--. 1 root root 2223 Mar 13 01:14 audit.yaml
-rw-r--r--. 1 root root 1003 Mar 12 10:14 audit.yaml.bak
-rw-r--r--. 1 root root  259 Mar 16 06:03 k8s-node.yaml
-rw-------. 1 root root 6178 Mar 16 06:00 kubelet.kubeconfig
-rw-------. 1 root root 6197 Mar 16 10:06 kube-proxy.kubeconfig
[root@pl66-242 conf]# scp kube-proxy.kubeconfig root@10.10.66.243:/opt/kubernetes/server/bin/conf/
root@10.10.66.243's password: 
kube-proxy.kubeconfig                                                                                                    100% 6197     2.3MB/s   00:00    
[root@pl66-242 conf]#

创建kube-proxy启动脚本

$\color{blue}{pl66-242上：}$

加载ipvs模块

/root/ipvs.sh

#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for i in $(ls $ipvs_mods_dir|grep -o "[^.]*")
do
  /sbin/modinfo -F filename $i &>/dev/null 
  if [ $? -eq 0 ];then
    /sbin/modprobe $i
  fi
done

增加执行权限

chmod +x ipvs.sh

执行脚本输出结果

[root@pl66-242 ~]# lsmod | grep ip_vs
ip_vs_wrr              12697  0 
ip_vs_wlc              12519  0 
ip_vs_sh               12688  0 
ip_vs_sed              12519  0 
ip_vs_rr               12600  0 
ip_vs_pe_sip           12740  0 
nf_conntrack_sip       33860  1 ip_vs_pe_sip
ip_vs_nq               12516  0 
ip_vs_lc               12516  0 
ip_vs_lblcr            12922  0 
ip_vs_lblc             12819  0 
ip_vs_ftp              13079  0 
ip_vs_dh               12688  0 
ip_vs                 145497  24 ip_vs_dh,ip_vs_lc,ip_vs_nq,ip_vs_rr,ip_vs_sh,ip_vs_ftp,ip_vs_sed,ip_vs_wlc,ip_vs_wrr,ip_vs_pe_sip,ip_vs_lblcr,ip_vs_lblc
nf_nat                 26787  3 ip_vs_ftp,nf_nat_ipv4,nf_nat_masquerade_ipv4
nf_conntrack          133095  8 ip_vs,nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_netlink,nf_conntrack_sip,nf_conntrack_ipv4
libcrc32c              12644  4 xfs,ip_vs,nf_nat,nf_conntrack

$\color{blue}{\#复制到pl66-243上：}$

LVS官方网站上的调度算法

轮叫调度（Round-Robin Scheduling）
加权轮叫调度（Weighted Round-Robin Scheduling）
最小连接调度（Least-Connection Scheduling）
加权最小连接调度（Weighted Least-Connection Scheduling）
基于局部性的最少链接（Locality-Based Least Connections Scheduling）
带复制的基于局部性最少链接（Locality-Based Least Connections with Replication Scheduling）
目标地址散列调度（Destination Hashing Scheduling）
源地址散列调度（Source Hashing Scheduling）
最短预期延时调度（Shortest Expected Delay Scheduling）
不排队调度（Never Queue Scheduling）

创建启动脚本

/opt/kubernetes/server/bin/kube-proxy.sh

[root@pl66-242 bin]# cat kube-proxy.sh 
#!/bin/bash
./kube-proxy \
    --cluster-cidr 172.16.0.0/16 \
    --hostname-override pl66-242.host.com \
    --proxy-mode=ipvs \
    --ipvs-scheduler=nq \
    --kubeconfig ./conf/kube-proxy.kubeconfig

注意：kube-proxy集群各主机的启动脚本略有不同，部署其他节点时注意修改。

检查配置，权限，创建日志目录

$\color{blue}{pl66-242上：}$

[root@pl66-242 bin]# ll conf/ | grep kube-proxy
-rw-------. 1 root root 6197 Mar 16 10:06 kube-proxy.kubeconfig
[root@pl66-242 bin]# chmod +x /opt/kubernetes/server/bin/kube-proxy.sh
[root@pl66-242 bin]# mkdir -p /data/logs/kubernetes/kube-proxy

创建supervisor配置

$\color{blue}{pl66-242上：}$

/etc/supervisord.d/kube-proxy.ini

[program:kube-proxy--66-242]
command=/opt/kubernetes/server/bin/kube-proxy.sh
numprocs=1
directory=/opt/kubernetes/server/bin/
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

查看proxy日志

[root@pl66-242 supervisord.d]# tail -fn 200 /data/logs/kubernetes/kube-proxy/proxy.stdout.log

安装ipvsadm

[root@pl66-242 supervisord.d]# yum -y install ipvsadm

验证kubernetes集群

在任意一个运算节点，创建一个资源配置清单

/root/nginx-ds.yaml

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: nginx-ds
spec:
  template:
    metadata:
      labels:
        app: nginx-ds
    spec:
      containers:
      - name: my-nginx
        image: harbor.yw.com/public/nginx:1.9.1
        ports:
        - containerPort: 80

应用资源配置，并检查

/root
[root@pl66-242 ~]# kubectl create -f nginx-ds.yaml

[root@pl66-242 ~]# kubectl get pods
NAME             READY   STATUS    RESTARTS   AGE
nginx-ds-mwp84   1/1     Running   0          <invalid>
nginx-ds-qck7g   1/1     Running   0          <invalid>
[root@pl66-242 ~]# 
[root@pl66-242 ~]# kubectl get pods -o wide
NAME             READY   STATUS    RESTARTS   AGE         IP             NODE                NOMINATED NODE   READINESS GATES
nginx-ds-mwp84   1/1     Running   0          <invalid>   172.16.242.2   pl66-242.host.com   <none>           <none>
nginx-ds-qck7g   1/1     Running   0          <invalid>   172.16.243.2   pl66-243.host.com   <none>           <none>
[root@pl66-242 ~]#

验证

[root@pl66-242 ~]# curl 172.16.242.2
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@pl66-242 ~]#

最后编辑于：2020.03.17 18:16:57

禁止转载，如需转载请通过简信或评论联系作者。

人面猴
序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
沈念sama阅读 204,684评论 6赞 478
死咒
序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
沈念sama阅读 87,143评论 2赞 381
救了他两次的神仙让他今天三更去死
文/潘晓璐我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
开封第一讲书人阅读 151,214评论 0赞 337
道士缉凶录：失踪的卖姜人
文/不坏的土叔我叫张陵，是天一观的道长。经常有香客问我，道长，这世上最难降的妖魔是什么？我笑而不...
开封第一讲书人阅读 54,788评论 1赞 277
港岛之恋（遗憾婚礼）
正文为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
茶点故事阅读 63,796评论 5赞 368
恶毒庶女顶嫁案：这布局不是一般人想出来的
文/花漫我一把揭开白布。她就那样静静地躺着，像睡着了一般。火红的嫁衣衬着肌肤如雪。梳的纹丝不乱的头发上，一...
开封第一讲书人阅读 48,665评论 1赞 281
城市分裂传说
那天，我揣着相机与录音，去河边找鬼。笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。我是一名探鬼主播，决...
沈念sama阅读 38,027评论 3赞 399
双鸳鸯连环套：你想象不到人心有多黑
文/苍兰香墨我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
开封第一讲书人阅读 36,679评论 0赞 258
万荣杀人案实录
序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
沈念sama阅读 41,346评论 1赞 299
护林员之死
正文独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋以下内容为张勋视角年9月15日...
茶点故事阅读 35,664评论 2赞 321
白月光启示录
正文我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
茶点故事阅读 37,766评论 1赞 331
活死人
序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
沈念sama阅读 33,412评论 4赞 321
日本核电站爆炸内幕
正文年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
茶点故事阅读 39,015评论 3赞 307
男人毒药：我在死后第九天来索命
文/蒙蒙一、第九天我趴在偏房一处隐蔽的房顶上张望。院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
开封第一讲书人阅读 29,974评论 0赞 19
一桩弑父案，背后竟有这般阴谋
文/苍兰香墨我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。一阵脚步声响...
开封第一讲书人阅读 31,203评论 1赞 260
情欲美人皮
我被黑心中介骗来泰国打工，没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。一个月前我还...
沈念sama阅读 45,073评论 2赞 350
代替公主和亲
正文我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
茶点故事阅读 42,501评论 2赞 343