首先用ansible 来部署ceph，这个是一个开源项目，文档在这里
https://docs.ceph.com/ceph-ansible/master/
以前用ceph-deploy 部署其实现在也可以，只是现在ansible这么流行，而且可以复用，维持鲁棒性，因此都在退ansible维护。
上面的内容待补充，因为原有官方文档够详细了，不过到时候贴几个配置的例子就行了
ceph 维护
首先是对ceph所有进程的维护
启动，停止和重新启动所有Ceph守护程
systemctl start ceph.target
systemctl stop ceph.target
systemctl restart ceph.target
按类型启动，停止和重新启动Ceph守护程序
在Ceph Monitor节点上：
[root@mon ~]# systemctl start ceph-mon.target
[root@mon ~]# systemctl stop ceph-mon.target
[root@mon ~]# systemctl restart ceph-mon.target
在Ceph Manager节点上：
[root@mgr ~]# systemctl start ceph-mgr.target
[root@mgr ~]# systemctl stop ceph-mgr.target
[root@mgr ~]# systemctl restart ceph-mgr.target
在Ceph OSD节点上：
Starting
[root@osd ~]# systemctl start ceph-osd.target
Stopping
[root@osd ~]# systemctl stop ceph-osd.target
Restarting
[root@osd ~]# systemctl restart ceph-osd.target
在Ceph Object Gateway节点上：
[root@rgw ~]# systemctl start ceph-radosgw.target
Stopping
[root@rgw ~]# systemctl stop ceph-radosgw.target
Restarting
[root@rgw ~]# systemctl restart ceph-radosgw.target
关闭或者重启ceph 集群
关闭的顺序是 先设置集群状态，关闭osd 服务，最后关闭mon 服务
启动的顺序是 先开启mon服务，然后开启osd服务，最后设置集群状态
#ceph fs set <fs_name> max_mds 1
#ceph mds deactivate <fs_name>:1 # rank 2 of 2
#ceph status # wait for rank 1 to finish stopping
#ceph fs set <fs_name> cluster_down true
#ceph mds fail <fs_name>:0
然后设置ceph 集群禁止调度恢复文件。
#ceph osd set noout
#ceph osd set norecover
#ceph osd set norebalance
#ceph osd set nobackfill
#ceph osd set nodown
#ceph osd set pause
然后按顺序关闭osd服务和mon服务，
启动是先开启mon服务，osd服务，然后重新设置集群状态
Power on the monitor nodes:
[root@mon ~]# systemctl start ceph-mon.target
Power on the OSD nodes:
[root@osd ~]# systemctl start ceph-osd.target
Wait for all the nodes to come up. Verify all the services are up and the connectivity is fine between the nodes.
Unset the noout, norecover, norebalance, nobackfill, nodown and pause flags. Run the following on a node with the client keyrings, for example, the Ceph Monitor or OpenStack controller node:
如果使用cephfs 最后还要重新设置集群状态
#ceph osd unset noout
#ceph osd unset norecover
#ceph osd unset norebalance
#ceph osd unset nobackfill
#ceph osd unset nodown
#ceph osd unset pause
[root@admin~]# ceph fs set <fs_name> cluster_down false
Start the RADOS Gateway and NFS-Ganesha Gateway.
On the RADOS Gateway node:
# systemctl start ceph-radosgw.target
On the NFS-Ganesha Gateway node:
# systemctl start nfs-ganesha.service
ceph 安装好后，
看状态基本就是几个命令
ceph health detail
ceph -s
ceph -w
root@proxmox01s:~# ceph health detail
HEALTH_OK
root@proxmox01s:~# ceph -s
cluster:
id: c3ad598e-2444-4d40-ba94-3001d4a0ccb7
health: HEALTH_OK
services:
mon: 3 daemons, quorum proxmox01s,proxmox02s,proxmox03s
mgr: proxmox03s(active), standbys: proxmox02s
mds: cephfs-1/1/1 up {0=proxmox04s=up:active}, 1 up:standby
osd: 40 osds: 40 up, 40 in
data:
pools: 4 pools, 1672 pgs
objects: 2.80M objects, 10.7TiB
usage: 32.1TiB used, 26.3TiB / 58.4TiB avail
pgs: 1669 active+clean
3 active+clean+scrubbing+deep
io:
client: 10.8MiB/s rd, 29.6MiB/s wr, 176op/s rd, 590op/s wr
root@proxmox01s:~# ceph -w
cluster:
id: c3ad598e-2444-4d40-ba94-3001d4a0ccb7
health: HEALTH_OK
services:
mon: 3 daemons, quorum proxmox01s,proxmox02s,proxmox03s
mgr: proxmox03s(active), standbys: proxmox02s
mds: cephfs-1/1/1 up {0=proxmox04s=up:active}, 1 up:standby
osd: 40 osds: 40 up, 40 in
data:
pools: 4 pools, 1672 pgs
objects: 2.80M objects, 10.7TiB
usage: 32.1TiB used, 26.3TiB / 58.4TiB avail
pgs: 1669 active+clean
3 active+clean+scrubbing+deep
io:
client: 9.15MiB/s rd, 26.1MiB/s wr, 148op/s rd, 535op/s wr
查看当前的pool
ceph osd pool ls
root@proxmox01s:~# ceph osd pool ls
vm-disk
cephfs_data
cephfs_metadata
ssd
查看pool里面的内容,注意命令不同，输出也不一样
rbd ls -p vm-disk
rados ls -p vm-disk
root@proxmox01s:~# rados ls -p vm-disk |more
rbd_data.484cf86b8b4567.000000000001bd76
rbd_data.6493626b8b4567.0000000000015622
rbd_data.4aa7406b8b4567.000000000001adfb
rbd_data.4859ad6b8b4567.000000000000fba7
rbd_data.4423106b8b4567.000000000000fc0c
rbd_data.4373b76b8b4567.0000000000012016
rbd_data.438d176b8b4567.000000000000e59d
rbd ls -p vm-disk
root@proxmox01s:~# rbd ls -p vm-disk
base-200-disk-0
kubernetes-dynamic-pvc-13bb4c14-2d24-11e9-81e1-da8820d8b83a
kubernetes-dynamic-pvc-2244af0b-31cf-11e9-9e13-56913ba006a2
kubernetes-dynamic-pvc-37b0028a-1b0f-11ea-8050-ea50e0a750db
kubernetes-dynamic-pvc-38c7186d-4946-11e9-b616-def7505fd032
kubernetes-dynamic-pvc-3b0c03ed-318f-11e9-9e13-56913ba006a2
kubernetes-dynamic-pvc-3e0654c6-38cc-11e9-ba72-def7505fd032
kubernetes-dynamic-pvc-3fc43857-43c6-11e9-b616-def7505fd032
kubernetes-dynamic-pvc-438e0a98-55eb-11e9-b616-def7505fd032
kubernetes-dynamic-pvc-4e9d3e7d-90da-11e9-bc53-def7505fd032
kubernetes-dynamic-pvc-56cd18ec-43c6-11e9-b616-def7505fd032
查看集群使用空间
root@proxmox01s:~# ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
58.4TiB 26.3TiB 32.1TiB 54.93
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
vm-disk 2 10.6TiB 67.28 5.16TiB 2787627
cephfs_data 6 47.7GiB 0.89 5.16TiB 12609
cephfs_metadata 7 187MiB 0 5.16TiB 75
ssd 8 0B 0 5.16TiB 0
