动态PVC可以简化管理员创建PV的过程，提升整个分布式存储的使用效率和体验;底层存储部署完成以后，用户只需要通过storageclass就可以申请pvc使用，减少管理员干预的过程
环境
| 角色 | 节点 |
|---|---|
| glusterfs 集群 | master-192,node-193（已部署） |
| k8s集群 | master-192,node-193,node-194（已部署） |
| hekti容器 | master-192（未部署） |
部署heketi
heketi提供RESTful接口与kubernetes对接，提供动态创建pvc的能力
由于glusterfs采用单独部署，故heketi工作采用ssh模式
git clone https://github.com/xiaotech/glusterfs_pvc.git
1.创建ssh免密钥认证key
ssh-keygen -f /etc/heketi/heketi_key -t rsa -N ''
[root@master-192 self]# ls /etc/heketi/ -l
总用量 164
-rw------- 1 root root 1679 11月 9 14:58 heketi_key
-rw-r--r-- 1 root root 397 11月 9 14:58 heketi_key.pub
2.创建serviceaccount
[root@master-192 self]# kubectl create -f heketi-service-account.json
apiVersion: v1
kind: ServiceAccount
metadata:
name: heketi-service-account
3.创建heketi配置文件
[root@master-192 self]# kubectl create cm heketi-config --from-file=./heketi.json
{
"_port_comment": "Heketi Server Port Number",
"port": "8080",
"_use_auth": "Enable JWT authorization. Please enable for deployment",
"use_auth": true,
"_jwt": "Private keys for access",
"jwt": {
"_admin": "Admin has access to all APIs",
"admin": {
"key": "xiaotech"
},
"_user": "User only has access to /volumes endpoint",
"user": {
"key": "My Secret"
}
},
"_glusterfs_comment": "GlusterFS Configuration",
"glusterfs": {
"_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
"executor": "ssh",
"_db_comment": "Database file name",
"db": "/var/lib/heketi/heketi.db",
"kubeexec": {
"rebalance_on_expansion": true
},
"sshexec": {
"rebalance_on_expansion": true,
"keyfile": "/etc/ssh_key/heketi_key",
"fstab": "/etc/fstab",
"port": "22",
"user": "root",
"sudo": false
}
},
"_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
"backup_db_to_kube_secret": false
}
4.创建heketi deployment和service
[root@master-192 self]# kubectl create -f heketi-deploy.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: heketi
spec:
replicas: 1
template:
metadata:
labels:
app: heketi
spec:
containers:
- env:
- name: HEKETI_EXECUTOR
value: ssh
- name: HEKETI_DB_PATH
value: /var/lib/heketi/heketi.db
- name: HEKETI_FSTAB
value: /var/lib/heketi/fstab
- name: HEKETI_SNAPSHOT_LIMIT
value: "14"
- name: HEKETI_KUBE_GLUSTER_DAEMONSET
value: "y"
image: xiaotech/heketi:7
livenessProbe:
failureThreshold: 3
httpGet:
path: /hello
port: 8080
scheme: HTTP
initialDelaySeconds: 30
timeoutSeconds: 3
name: heketi
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /hello
port: 8080
scheme: HTTP
initialDelaySeconds: 3
timeoutSeconds: 3
volumeMounts:
- mountPath: /var/lib/heketi
name: db
- mountPath: /etc/ssh_key
name: key
- mountPath: /etc/heketi/
name: config
nodeSelector:
kubernetes.io/hostname: master-192
serviceAccount: heketi-service-account
serviceAccountName: heketi-service-account
volumes:
- emptyDir: {}
name: db
- hostPath:
path: /etc/heketi/
name: key
- configMap:
name: heketi-config
name: config
[root@master-192 self]# kubectl create -f heketi-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: heketi
name: heketi
namespace: default
spec:
selector:
app: heketi
ports:
- port: 8080
type: NodePort
5.验证
[root@master-192 self]# heketi-cli --server http://172.30.81.192:31131 --user=admin --secret=xiaotech cluster list
Clusters:
6.创建topology
[root@master-192 home]# heketi-cli --server http://172.30.81.192:31131 --user=admin --secret=xiaotech topology load --json ./heketi-topology.json
Found node 172.30.81.192 on cluster f9256fd3b71b47cfdaff71d605ce2766
Adding device /dev/vdb ... OK
Found node 172.30.81.193 on cluster f9256fd3b71b47cfdaff71d605ce2766
Adding device /dev/vdb ... OK
7.创建storageclass
[root@master-192 self]# kubectl create -f storageclass-glusterfs.yaml
storageclass.storage.k8s.io/glusterfs created
8.创建动态pvc
[root@master-192 self]# kubectl create -f glusterfs-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: test
namespace: default
annotations:
volume.beta.kubernetes.io/storage-class: "glusterfs"
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 2Gi
[root@master-192 self]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
vda 252:0 0 40G 0 disk
├─vda1 252:1 0 1G 0 part /boot
└─vda2 252:2 0 39G 0 part
├─centos-root 253:0 0 35G 0 lvm /
└─centos-swap 253:1 0 4G 0 lvm
vdb 252:16 0 20G 0 disk
├─vg_8f9e89da702ee20d753162ee528de5cc-tp_1dab4f6ca2aa9932a56fc9b8f1209abf_tmeta 253:2 0 12M 0 lvm
│ └─vg_8f9e89da702ee20d753162ee528de5cc-tp_1dab4f6ca2aa9932a56fc9b8f1209abf-tpool 253:4 0 2G 0 lvm
│ ├─vg_8f9e89da702ee20d753162ee528de5cc-tp_1dab4f6ca2aa9932a56fc9b8f1209abf 253:5 0 2G 0 lvm
│ └─vg_8f9e89da702ee20d753162ee528de5cc-brick_1dab4f6ca2aa9932a56fc9b8f1209abf 253:6 0 2G 0 lvm /var/lib/heketi/mounts/vg_8
└─vg_8f9e89da702ee20d753162ee528de5cc-tp_1dab4f6ca2aa9932a56fc9b8f1209abf_tdata 253:3 0 2G 0 lvm
└─vg_8f9e89da702ee20d753162ee528de5cc-tp_1dab4f6ca2aa9932a56fc9b8f1209abf-tpool 253:4 0 2G 0 lvm
├─vg_8f9e89da702ee20d753162ee528de5cc-tp_1dab4f6ca2aa9932a56fc9b8f1209abf 253:5 0 2G 0 lvm
└─vg_8f9e89da702ee20d753162ee528de5cc-brick_1dab4f6ca2aa9932a56fc9b8f1209abf 253:6 0 2G 0 lvm /var/lib/heketi/mounts/vg_8
[root@master-192 self]# gluster volume info
Volume Name: vol_8449601e8b59a3ea1ac5dc9c0e76fbe5
Type: Replicate
Volume ID: 0b948972-0bb2-4887-9839-7f4161f65227
Status: Started
Snapshot Count: 0
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: 172.30.81.193:/var/lib/heketi/mounts/vg_6b1b9f0c4532e4b2e2347a5861d01beb/brick_2164a574204385f2e9b03d50dd7cc91a/brick
Brick2: 172.30.81.192:/var/lib/heketi/mounts/vg_8f9e89da702ee20d753162ee528de5cc/brick_1dab4f6ca2aa9932a56fc9b8f1209abf/brick
Options Reconfigured:
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off
删除pvc
[root@master-192 self]# kubectl delete -f glusterfs-pvc.yaml
[root@master-192 self]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
vda 252:0 0 40G 0 disk
├─vda1 252:1 0 1G 0 part /boot
└─vda2 252:2 0 39G 0 part
├─centos-root 253:0 0 35G 0 lvm /
└─centos-swap 253:1 0 4G 0 lvm
vdb 252:16 0 20G 0 disk
[root@master-192 self]# gluster volume info
No volumes present
