收到一个需求，快速地在一个完全不通外网的服务器利用minikube安装单机版本k8s，由于服务器无外网，还是遇到了很多曲折，特此总结一下，以备自己和他人参考。
- 配置离线yum源，离线安装docker-ce，此处安装的版本为19.03.9，离线yum源步骤参考https://www.jianshu.com/p/821ac391b7a9
# 安装docker-ce
yum install -y docker-ce
# 创建docker的镜像存放目录，以防默认磁盘空间不够用
mkdir /home/finance/docker_home
# 修改默认docker根目录
mkdir /etc/docker
echo '{
"data-root": "/home/finance/docker_home"
}' > /etc/docker/daemon.json
#给某用户添加docke权限
usermod -aG docker finance
# 启动docker-ce并设置自启动
systemctl enable --now docker
docker info
- 关闭系统交换区
swapoff -a
- 安装docker-compose
# 下载 docker-compose-Linux-x86_64
# 上传到/usr/local/bin
[root@ai-server-test-01 bin]# ls -ltr
total 10608
-rw-r--r-- 1 root root 10858808 Nov 12 10:09 docker-compose-Linux-x86_64
[root@ai-server-test-01 bin]# pwd
/usr/local/bin
# 重命名并修改权限
cp docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
docker-compose version
# 版本如下
[root@ai-server-test-01 bin]# docker-compose version
docker-compose version 1.21.2, build a133471
docker-py version: 3.3.0
CPython version: 3.6.5
OpenSSL version: OpenSSL 1.0.1t 3 May 2016
- 安装harbor，参考 https://www.jianshu.com/p/f7eb62b31380，并把k8s需要的镜像上传到harbor
- docker登陆harbor，以便后续upload
docker login -u admin -p <password> harbor.ai.com
# 需要的k8s镜像如下
storage-provisioner v1.8.1
kube-proxy v1.18.3
kube-apiserver v1.18.3
kube-scheduler v1.18.3
kube-controller-manager v1.18.3
pause 3.2
coredns 1.6.7
etcd 3.4.3-0
- 安装minikube
# 上传到服务器，并更改权限，重命名
chmod +x minikube-linux-x86_64
cp minikube-linux-x86_64 /usr/local/bin/minikube
# 版本如下
[root@ai-server-test-01 bin]# minikube version
minikube version: v1.11.0
commit: 57e2f55f47effe9ce396cea42a1e0eb4f611ebbd
- 安装，并启动单节点k8s
v1.18.3
# 安装conntrack
yum install conntrack -y
# 由于是纯离线，无法从网上拉取k8s的包，所以需要手动创建好以下目录 （选做，如果找不到kubectl的的时候）
mkdir -p /root/.minikube/cache/linux/v1.18.3
# 上传已经下载好的对应版本执行文件 （选做，如果找不到kubectl的的时候）
kubeadm kubectl kubelet
# 把提前下载好的kubeadm kubectl kubelet放置到/usr/local/bin并加执行权限
kubeadm
kubectl
kubelet
#版本如下
[root@ai-server-test-01 bin]# kubectl version
Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.3", GitCommit:"2e7996e3e2712684bc73f0dec0200d64eec7fe40", GitTreeState:"clean", BuildDate:"2020-05-20T12:52:00Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.3", GitCommit:"2e7996e3e2712684bc73f0dec0200d64eec7fe40", GitTreeState:"clean", BuildDate:"2020-05-20T12:43:34Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}
#启动minikube
# --vm-driver=none，本来就是虚拟机，无需再虚拟一个。
minikube start --kubernetes-version v1.18.3 --vm-driver=none --image-repository=harbor.ai.com/kube --insecure-registry=harbor.ai.com
minikube status
minikube logs
# 运行结果参考以下
[root@ai-server-test-01 software]# minikube start --kubernetes-version v1.18.3 --vm-driver=none --image-repository=harbor.ai.com/kube --insecure-registry=harbor.ai.com
* minikube v1.11.0 on Centos 7.6.1810
* Using the none driver based on existing profile
* Starting control plane node minikube in cluster minikube
* Restarting existing none bare metal machine for "minikube" ...
* OS release is CentOS Linux 7 (Core)
* Preparing Kubernetes v1.18.3 on Docker 19.03.9 ...
* Configuring local host environment ...
*
! The 'none' driver is designed for experts who need to integrate with an existing VM
* Most users should use the newer 'docker' driver instead, which does not require root!
* For more information, see: https://minikube.sigs.k8s.io/docs/reference/drivers/none/
*
! kubectl and minikube configuration will be stored in /root
! To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:
*
- sudo mv /root/.kube /root/.minikube $HOME
- sudo chown -R $USER $HOME/.kube $HOME/.minikube
*
* This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_USER=true
* Verifying Kubernetes components...
* Enabled addons: default-storageclass, storage-provisioner
* Done! kubectl is now configured to use "minikube"
* For best results, install kubectl: https://kubernetes.io/docs/tasks/tools/install-kubectl/
[root@ai-server-test-01 software]# minikube status
minikube
type: Control Plane
host: Running
kubelet: Running
apiserver: Running
kubeconfig: Configured
# 确保所有pods都启动正常
[root@ai-server-test-01 software]# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
default nginx-app-684bd4f4c-mkvdm 1/1 Running 0 45m
default nginx-app-684bd4f4c-wnrfn 1/1 Running 0 45m
kube-system coredns-6bb6b45975-dbqbw 1/1 Running 1 88m
kube-system coredns-6bb6b45975-lv4c8 1/1 Running 1 88m
kube-system etcd-ai-server-test-01 1/1 Running 1 88m
kube-system kube-apiserver-ai-server-test-01 1/1 Running 1 88m
kube-system kube-controller-manager-ai-server-test-01 1/1 Running 1 88m
kube-system kube-proxy-7hmvj 1/1 Running 1 88m
kube-system kube-scheduler-ai-server-test-01 1/1 Running 1 88m
kube-system storage-provisioner 1/1 Running 0 88m
- 测试k8s集群
# 需要保证nginx镜像已经在minikube设置的仓库中
# 创建deployment的配置文件
# API 版本号
apiVersion: apps/v1
# 类型，如：Pod/ReplicationController/Deployment/Service/Ingress
kind: Deployment
metadata:
# Kind 的名称
name: nginx-app
spec:
selector:
matchLabels:
# 容器标签的名字，发布 Service 时，selector 需要和这里对应
app: nginx
# 部署的实例数量
replicas: 2
template:
metadata:
labels:
app: nginx
spec:
# 配置容器，数组类型，说明可以配置多个容器
containers:
# 容器名称
- name: nginx
# 容器镜像
image: nginx:latest
# 只有镜像不存在时，才会进行镜像拉取
imagePullPolicy: IfNotPresent
ports:
# Pod 端口
- containerPort: 80
# 执行命令，创建deployment
kubectl apply -f nginx.yml
# 查看pods
[root@ai-server-test-01 software]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-app-684bd4f4c-mkvdm 1/1 Running 0 4s
nginx-app-684bd4f4c-wnrfn 1/1 Running 0 4s
# 使用expose命令将pod暴露到外部环境
[root@ai-server-test-01 software]# kubectl expose deployment nginx-app --type=LoadBalancer
service/nginx-app exposed
# 查看刚创建的services
[root@ai-server-test-01 software]# kubectl get services
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 51m
nginx-app LoadBalancer 10.103.29.3 <pending> 80:32457/TCP 5m27s
# 通过--type=LoadBalancer来将service暴露到集群外，在支持负载的云平台，将配置外部IP地址来访问service，此处显示为<pending>,在minikube上，可以使用minikube service来查看
[root@ai-server-test-01 software]# minikube service nginx-app
|-----------|-----------|-------------|----------------------------|
| NAMESPACE | NAME | TARGET PORT | URL |
|-----------|-----------|-------------|----------------------------|
| default | nginx-app | 80 | http://10.193.199.91:32457 |
|-----------|-----------|-------------|----------------------------|
* Opening service default/nginx-app in default browser...
xdg-open: no method available for opening 'http://10.193.199.91:32457'
*
X open url failed: http://10.193.199.91:32457: exit status 3
*
* minikube is exiting due to an error. If the above message is not useful, open an issue:
- https://github.com/kubernetes/minikube/issues/new/choose
#可以curl一下，看看nginx是否启动
[root@ai-server-test-01 software]# curl http://10.193.199.91:32457
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
# 测试nginx运行正常
- minikube命令
minikube stop
minikube service
minikube help
