一、假设你已经拥有一个可用使用得ceph分布式存储系统，以及k8s平台
安装参考：http://docs.ceph.org.cn/rados/（安装mds）
基本环境
k81集群1.13.1版本
[root@elasticsearch01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
10.2.9.30 Ready <none> 25d v1.13.1
10.2.9.31 Ready <none> 25d v1.13.1
ceph集群 luminous版本
[root@ceph01 ~]# ceph -s
services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03
mgr: ceph03(active), standbys: ceph02, ceph01
osd: 24 osds: 24 up, 24 in
rgw: 3 daemons active
二、创建CEPH 文件系统，名称cephfs
1、要用默认设置为文件系统创建两个存储池
ceph osd pool create cephfs_data 1024 1024
ceph osd pool create cephfs_metadata 1024 1024
2、创建文件系统
ceph fs new cephfs cephfs_metadata cephfs_data
ceph fs ls
name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
ceph mds stat
cephfs-1/1/1 up {0=k8s-node2=up:active}, 2 up:standby
三：创建使用认证证书
ceph auth get-key client.admin |base64
QVFBbU9ZSmNUSWQ3TlJBQVhKeWh3c2ZtQkhzQzZ2VGJ4UVZvVWc9PQ==
cat ceph-secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: cephfs-secret
data:
key: QVFBbU9ZSmNUSWQ3TlJBQVhKeWh3c2ZtQkhzQzZ2VGJ4UVZvVWc9PQ==
四、 Kubernetes StorageClass使用CephFS
使用ceph cephfs创建pv
cat jenkins-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-home-pv
spec:
capacity:
storage: 40Gi
accessModes:
- ReadWriteMany
cephfs:
monitors:
- '10.0.4.10:6789'
- '10.0.4.11:6789'
- '10.0.4.12:6789'
user: admin
secretRef:
name: cephfs-secret
readOnly: false
persistentVolumeReclaimPolicy: Recycle
2、创建pvc
[root@elasticsearch01 jenkins]# cat jenkins-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins-home-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
[root@elasticsearch01 jenkins]# kubectl create -f jenkins-pvc.yaml
persistentvolumeclaim/jenkins-home-pvc created
[root@elasticsearch01 jenkins]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
jenkins-home-pvc Bound jenkins-home-pv 40Gi RWO 3s
[root@elasticsearch01 jenkins]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
jenkins-home-pv 40Gi RWO Recycle Bound default/jenkins-home-pvc
五：部署最新版本jenkins
参考https://github.com/jenkinsci/kubernetes-plugin/tree/master/src/main/kubernetes
需要修改得地方：
二、跟进实际情况修改jenkins.yml文件
主要修改的配置从上到下分别是： 1、拉取镜像策略
imagePullPolicy: IfNotPresent
2、自动存储storage class改成voulumes的pvc方式实现
volumes:
- name: jenkins-home
persistentVolumeClaim:
claimName: jenkins-home-pvc
3、ingress的host改成实际的
host: jenkins.search.com
4、ingres的tls证书改成实际的
tls:
- hosts:
- jenkins.search.com
secretName: ingress-secret
5、需要修改（创建提示没有权限问题）
securityContext:
runAsUser: 0
6、具体如下
[root@ jenkins]# cat jenkins.yml
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: jenkins
labels:
name: jenkins
spec:
serviceName: jenkins
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: jenkins
containers:
- name: jenkins
image: jenkins/jenkins:lts-alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
- containerPort: 50000
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 0.5
memory: 500Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
# value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:
- name: jenkins-home
mountPath: /var/jenkins_home
readOnly: false
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
securityContext:
runAsUser: 0
volumes:
- name: jenkins-home
persistentVolumeClaim:
claimName: jenkins-home-pvc
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
# type: LoadBalancer
selector:
name: jenkins
# ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)
#externalTrafficPolicy: Local
ports:
-
name: http
port: 80
targetPort: 8080
protocol: TCP
-
name: agent
port: 50000
protocol: TCP
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: jenkins
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
kubernetes.io/tls-acme: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
nginx.ingress.kubernetes.io/proxy-body-size: 50m
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
# For nginx-ingress controller < 0.9.0.beta-18
ingress.kubernetes.io/ssl-redirect: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
ingress.kubernetes.io/proxy-body-size: 50m
ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
rules:
- http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 80
host: jenkins.search.com
tls:
- hosts:
- jenkins.search.com
secretName: ingress-secret
三、创建状态集、svc、pod、ingress
1、创建rbac认证角色
[root@ jenkins]# kubectl create -f service-account.yml
serviceaccount/jenkins created
role.rbac.authorization.k8s.io/jenkins created
rolebinding.rbac.authorization.k8s.io/jenkins created
2、创建jenkins服务等
[root@ jenkins]# kubectl create -f jenkins.yml
statefulset.apps/jenkins created
service/jenkins created
ingress.extensions/jenkins created 4s
[root@elasticsearch01 jenkins]# kubectl get pods
NAME READY STATUS RESTARTS AGE
jenkins-0 0/1 ContainerCreating 0 7s
[root@ jenkins]# kubectl get pods
NAME READY STATUS RESTARTS AGE
jenkins-0 1/1 Running 0 4m52s
四、通过ingress访问
获取ingress-nginx对外端口，https://jenkins.search.com:30887/访问即可，需要配置dns解析到pod所在node的ip
[root@ jenkins]# kubectl get svc -n ingress-nginx|grep ingress-nginx
ingress-nginx LoadBalancer 10.254.43.251 <pending> 80:32827/TCP,443:30887/TCP 3d19h
五、初始化jenkins
1、查找密码
[root@k8s-node1 ]# cd /var/lib/kubelet/pods/34aca452-4641-11e9-8b2c-089e010da283/volume-subpaths/jenkins-home/jenkins/0
[root@k8s-node1 0]# ls
config.xml jenkins.CLI.xml nodeMonitors.xml secrets
copy_reference_file.log jenkins.install.UpgradeWizard.state nodes updates
hudson.model.UpdateCenter.xml jenkins.telemetry.Correlator.xml plugins userContent
identity.key.enc jobs secret.key users
init.groovy.d logs secret.key.not-so-secret war
[root@k8s-node1 0]# cat secrets/initialAdminPassword
cf9964ff5c8c40878e31d040ae90d9a7
2、选择安装插件
3、创建初始管理账号
4、设置jenkins url默认https://jenkins.search.com:30887
5、开始使用jenkins
6、jenkins控制台界面，主要配置都在系统管理中
使用ceph rbd 这种只能读写一次的设备不能用在线上，线上应该使用分布式存储例如nfs，cephfs，glusterfs等，这里只是测试jenkins结合ceph，pv，pvc完成有状态pod的测试
