1. 部署环境
- Linux: Cent OS7
- K8s: v1.19.3
- Docker: 19.03
- Rook: v1.5.1
- 三个K8s 工作节点
2. rook使用限制
rook 自动在所有节点上启动 osd 设备，rook 会自动发现可用设备，必须保证：
- 设备没有分区
- 设备没有格式化的文件系统
- 集群至少三个节点有可用设备
3. 使用 helm3 安装 rook-ceph
# helm3 下载地址
https://github.com/helm/helm/releases
# 添加 helm 仓库
helm repo add rook-release https://charts.rook.io/release
# 新建 csi 配置文件（修改csi镜像地址为国内）
cat > csi-conf.yaml << EOF
csi:
registrar:
image: raspbernetes/csi-node-driver-registrar2.0.1
provisioner:
image: raspbernetes/csi-external-provisioner2.0.0
snapshotter:
image: raspbernetes/csi-external-snapshotter2.1.1
attacher:
image: raspbernetes/csi-external-attacher3.0.0
resizer:
image: raspbernetes/csi-external-resizer1.0.0
EOF
# 创建 namespace
kubectl create namespace rook-ceph
# 安装 rook-ceph
helm install rook-ceph rook-release/rook-ceph -f csi-conf.yaml -n rook-ceph
4. 安装 rook-ceph 以及工具包
# 下载 rook 项目
git clone --single-branch --branch v1.5.1 https://github.com/rook/rook.git
# 切换到 ceph 目录
cd rook/cluster/examples/kubernetes/ceph
# 安装 rook-ceph 集群
kubectl create -f cluster.yaml
# 安装 rook 工具包
kubectl create -f toolbox.yaml
5. 验证安装
Running 状态为安装完成
# 查看 pod 状态
$ kubectl -n rook-ceph get pod
NAME READY STATUS RESTARTS AGE
csi-cephfsplugin-provisioner-d77bb49c6-n5tgs 5/5 Running 0 140s
csi-cephfsplugin-provisioner-d77bb49c6-v9rvn 5/5 Running 0 140s
csi-cephfsplugin-rthrp 3/3 Running 0 140s
csi-rbdplugin-hbsm7 3/3 Running 0 140s
csi-rbdplugin-provisioner-5b5cd64fd-nvk6c 6/6 Running 0 140s
csi-rbdplugin-provisioner-5b5cd64fd-q7bxl 6/6 Running 0 140s
rook-ceph-crashcollector-minikube-5b57b7c5d4-hfldl 1/1 Running 0 105s
rook-ceph-mgr-a-64cd7cdf54-j8b5p 1/1 Running 0 77s
rook-ceph-mon-a-694bb7987d-fp9w7 1/1 Running 0 105s
rook-ceph-mon-b-856fdd5cb9-5h2qk 1/1 Running 0 94s
rook-ceph-mon-c-57545897fc-j576h 1/1 Running 0 85s
rook-ceph-operator-85f5b946bd-s8grz 1/1 Running 0 92m
rook-ceph-osd-0-6bb747b6c5-lnvb6 1/1 Running 0 23s
rook-ceph-osd-1-7f67f9646d-44p7v 1/1 Running 0 24s
rook-ceph-osd-2-6cd4b776ff-v4d68 1/1 Running 0 25s
rook-ceph-osd-prepare-node1-vx2rz 0/2 Completed 0 60s
rook-ceph-osd-prepare-node2-ab3fd 0/2 Completed 0 60s
rook-ceph-osd-prepare-node3-w4xyz 0/2 Completed 0 60s
rook-ceph-tools-76c7d559b6-8w7bk 1/1 Running 0 60s
# 查看 ceph 状态
$ kubectl -n rook-ceph exec -it rook-ceph-tools-76c7d559b6-8w7bk sh -c 'ceph status'
6. 登录 ceph dashboard
# 修改 dashboard service为 NodePort 类型
kubectl apply -f rook/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml
# 查看 dashboard 端口
kubectl get svc -n rook-ceph | grep dashboard
# 获取 dashboard 用户名与密码
MGR_POD=`kubectl get pod -n rook-ceph | grep mgr | awk '{print $1}'`
kubectl -n rook-ceph logs $MGR_POD | grep password
7. 安装 wordpress 并使用 ceph 集群存储数据
# 创建 rbd 类型 storageclass
kubectl apply -f rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
# 创建 mysql 服务
kubectl apply -f rook/cluster/examples/kubernetes/mysql.yaml
# 创建 wordpress 服务
kubectl apply -f rook/cluster/examples/kubernetes/wordpress.yaml
# 查看 pvc 绑定状态
kubectl get pvc
# 查看 wordpress 端口
kubectl get svc wordpress
# 访问 wordpress
8. 问题解答
# rook-ceph-crash-collector secret无法挂载问题
MountVolume.SetUp failed for volume "rook-ce │
│ ph-crash-collector-keyring" : secret "rook-ceph-crash-collector-keyring" not found
# 解决
1. 删除所有节点 /var/lib/rook 目录
2. 重新部署 kubectl apply -f cluser.yaml
