云原生第三课作业

一、COREDNS

1.1yml文件

root@master1:~/maniftest/coredns# cat coredns.yaml

# __MACHINE_GENERATED_WARNING__

apiVersion: v1

kind: ServiceAccount

metadata:

namespace: kube-system

labels:

kubernetes.io/cluster-service: "true"

addonmanager.kubernetes.io/mode: Reconcile

---

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRole

metadata:

labels:

kubernetes.io/bootstrapping: rbac-defaults

addonmanager.kubernetes.io/mode: Reconcile

rules:

- apiGroups:

- ""

resources:

- endpoints

- services

- pods

- namespaces

verbs:

- list

- watch

- apiGroups:

- ""

resources:

- nodes

verbs:

- get

- apiGroups: #后添加

- discovery.k8s.io #后添加

resources: #后添加

- endpointslices #后添加

verbs: #后添加

- list #后添加

- watch #后添加

---

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRoleBinding

metadata:

annotations:

rbac.authorization.kubernetes.io/autoupdate: "true"

labels:

kubernetes.io/bootstrapping: rbac-defaults

addonmanager.kubernetes.io/mode: EnsureExists

roleRef:

apiGroup: rbac.authorization.k8s.io

kind: ClusterRole

subjects:

- kind: ServiceAccount

namespace: kube-system

---

apiVersion: v1

kind: ConfigMap

metadata:

namespace: kube-system

labels:

addonmanager.kubernetes.io/mode: EnsureExists

data:

Corefile: |

.:53 {

errors

health {

lameduck 5s

}

ready

kubernetes fchiaas.local in-addr.arpa ip6.arpa { #fchiaas.local是k8s集群的名称

pods insecure

fallthrough in-addr.arpa ip6.arpa

ttl 30

}

prometheus :9153

forward . 10.0.0.20 { #10.0.0.20公司内部DNS

max_concurrent 1000

}

cache 30

loop

reload

loadbalance

}

---

apiVersion: apps/v1

kind: Deployment

metadata:

namespace: kube-system

labels:

k8s-app: kube-dns

kubernetes.io/cluster-service: "true"

addonmanager.kubernetes.io/mode: Reconcile

kubernetes.io/name: "CoreDNS"

spec:

# replicas: not specified here:

# 1. In order to make Addon Manager do not reconcile this replicas parameter.

# 2. Default is 1.

# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.

strategy:

type: RollingUpdate

rollingUpdate:

maxUnavailable: 1

selector:

matchLabels:

k8s-app: kube-dns

template:

metadata:

labels:

k8s-app: kube-dns

spec:

securityContext:

seccompProfile:

type: RuntimeDefault

priorityClassName: system-cluster-critical

serviceAccountName: coredns

affinity:

podAntiAffinity:

preferredDuringSchedulingIgnoredDuringExecution:

- weight: 100

podAffinityTerm:

labelSelector:

matchExpressions:

- key: k8s-app

operator: In

values: ["kube-dns"]

topologyKey: kubernetes.io/hostname

tolerations:

- key: "CriticalAddonsOnly"

operator: "Exists"

nodeSelector:

kubernetes.io/os: linux

containers:

- name: coredns

image: coredns/coredns:v1.8.6

imagePullPolicy: IfNotPresent

resources:

limits:

memory: 256Mi

requests:

cpu: 100m

memory: 70Mi

args: [ "-conf", "/etc/coredns/Corefile" ]

volumeMounts:

- name: config-volume

mountPath: /etc/coredns

readOnly: true

ports:

- containerPort: 53

protocol: UDP

- containerPort: 53

protocol: TCP

- containerPort: 9153

protocol: TCP

livenessProbe:

httpGet:

path: /health

port: 8080

scheme: HTTP

initialDelaySeconds: 60

timeoutSeconds: 5

successThreshold: 1

failureThreshold: 5

readinessProbe:

httpGet:

path: /ready

port: 8181

scheme: HTTP

securityContext:

allowPrivilegeEscalation: false

capabilities:

add:

- NET_BIND_SERVICE

drop:

- all

readOnlyRootFilesystem: true

dnsPolicy: Default

volumes:

- name: config-volume

configMap:

items:

- key: Corefile

path: Corefile

---

apiVersion: v1

kind: Service

metadata:

namespace: kube-system

annotations:

prometheus.io/port: "9153"

prometheus.io/scrape: "true"

labels:

k8s-app: kube-dns

kubernetes.io/cluster-service: "true"

addonmanager.kubernetes.io/mode: Reconcile

kubernetes.io/name: "CoreDNS"

spec:

selector:

k8s-app: kube-dns

clusterIP: 10.100.0.2 #10.0.0.2 是coredns服务的IP地址

ports:

- name: dns

port: 53

protocol: UDP

- name: dns-tcp

port: 53

protocol: TCP

- name: metrics

port: 9153

protocol: TCP

1.2部署

root@master1:~/maniftest/coredns# kubectl apply -f coredns.yaml

serviceaccount/coredns created

clusterrole.rbac.authorization.k8s.io/system:coredns created

clusterrolebinding.rbac.authorization.k8s.io/system:coredns created

configmap/coredns created

deployment.apps/coredns created

service/kube-dns created

root@master1:~/maniftest/coredns# kubectl get pods -n kube-system -w

NAME READY STATUS RESTARTS AGE

calico-kube-controllers-59df8b6856-qll6h 1/1 Running 4 (144m ago) 11d

calico-node-74687 1/1 Running 4 (144m ago) 11d

calico-node-f98lx 1/1 Running 4 (144m ago) 11d

calico-node-j2jhv 1/1 Running 4 (144m ago) 11d

calico-node-ljnwb 1/1 Running 7 (5d3h ago) 11d

coredns-69d84cdc49-rtpnd 1/1 Running 0 5m

coredns-69d84cdc49-xk7nq 1/1 Running 0 2m36s

root@master1:~/maniftest/coredns# kubectl get svc -n kube-system

NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE

kube-dns ClusterIP 10.100.0.2 <none> 53/UDP,53/TCP,9153/TCP 13m

root@master1:~/maniftest/coredns# kubectl get svc -A

NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE

default kubernetes ClusterIP 10.100.0.1 <none> 443/TCP 11d

kube-system kube-dns ClusterIP 10.100.0.2 <none> 53/UDP,53/TCP,9153/TCP 13m

1.3测试

root@master1:~/maniftest/coredns# kubectl exec -it test -- /bin/sh

sh-4.4# ping kube-dns

ping: kube-dns: Name or service not known

sh-4.4# ping kube-dns.kube-system

PING kube-dns.kube-system.svc.fchiaas.local (10.100.0.2) 56(84) bytes of data.

64 bytes from kube-dns.kube-system.svc.fchiaas.local (10.100.0.2): icmp_seq=1 ttl=64 time=0.036 ms

64 bytes from kube-dns.kube-system.svc.fchiaas.local (10.100.0.2): icmp_seq=2 ttl=64 time=0.045 ms

--- kube-dns.kube-system.svc.fchiaas.local ping statistics ---

2 packets transmitted, 2 received, 0% packet loss, time 1002ms

rtt min/avg/max/mdev = 0.036/0.040/0.045/0.007 ms

sh-4.4# ping kube-dns.kube-system.svc.fchiaas.local

PING kube-dns.kube-system.svc.fchiaas.local (10.100.0.2) 56(84) bytes of data.

64 bytes from kube-dns.kube-system.svc.fchiaas.local (10.100.0.2): icmp_seq=1 ttl=64 time=0.024 ms

64 bytes from kube-dns.kube-system.svc.fchiaas.local (10.100.0.2): icmp_seq=2 ttl=64 time=0.047 ms

--- kube-dns.kube-system.svc.fchiaas.local ping statistics ---

2 packets transmitted, 2 received, 0% packet loss, time 1013ms

rtt min/avg/max/mdev = 0.024/0.035/0.047/0.012 ms

二、dashboard

2.1下载yml文件

wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.4.0/aio/deploy/recommended.yaml -o dashbord-2.4.0.yaml

修改文件

spec:

type: NodePort #后添加

ports:

- port: 443

targetPort: 8443

nodePort: 30443 #后添加

2.2创建

root@master1:~/maniftest/dashboard# kubectl apply -f dashbord-2.4.0.yaml

namespace/kubernetes-dashboard created

serviceaccount/kubernetes-dashboard created

service/kubernetes-dashboard created

secret/kubernetes-dashboard-certs created

secret/kubernetes-dashboard-csrf created

secret/kubernetes-dashboard-key-holder created

configmap/kubernetes-dashboard-settings created

role.rbac.authorization.k8s.io/kubernetes-dashboard created

clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created

rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created

clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created

deployment.apps/kubernetes-dashboard created

service/dashboard-metrics-scraper created

deployment.apps/dashboard-metrics-scraper created

root@master1:~/maniftest/dashboard# kubectl get pods -A -o wide -w

NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES

default deploy-nginx-6899665dbb-2j2d6 1/1 Running 2 (3h6m ago) 5d11h 10.200.104.32 10.0.0.25 <none> <none>

default deploy-nginx-6899665dbb-8m27p 1/1 Running 2 (3h6m ago) 5d23h 10.200.104.31 10.0.0.25 <none> <none>

default deploy-nginx-6899665dbb-bpk6x 1/1 Running 2 (3h6m ago) 5d23h 10.200.104.34 10.0.0.25 <none> <none>

default deploy-nginx-6899665dbb-g7fms 1/1 Running 2 (3h6m ago) 5d23h 10.200.104.33 10.0.0.25 <none> <none>

default deploy-nginx-6899665dbb-q6btz 1/1 Running 2 (3h6m ago) 5d23h 10.200.104.36 10.0.0.25 <none> <none>

default deploy-nginx-6899665dbb-tzwbq 1/1 Running 2 (3h6m ago) 5d11h 10.200.104.35 10.0.0.25 <none> <none>

default test 1/1 Running 0 36m 10.200.166.146 10.0.0.24 <none> <none>

kube-system calico-kube-controllers-59df8b6856-qll6h 1/1 Running 4 (3h6m ago) 11d 10.0.0.25 10.0.0.25 <none> <none>

kube-system calico-node-74687 1/1 Running 4 (3h6m ago) 11d 10.0.0.22 10.0.0.22 <none> <none>

kube-system calico-node-f98lx 1/1 Running 4 (3h6m ago) 11d 10.0.0.24 10.0.0.24 <none> <none>

kube-system calico-node-j2jhv 1/1 Running 4 (3h6m ago) 11d 10.0.0.25 10.0.0.25 <none> <none>

kube-system calico-node-ljnwb 1/1 Running 7 (5d4h ago) 11d 10.0.0.21 10.0.0.21 <none> <none>

kube-system coredns-69d84cdc49-rtpnd 1/1 Running 0 46m 10.200.104.37 10.0.0.25 <none> <none>

kube-system coredns-69d84cdc49-xk7nq 1/1 Running 0 44m 10.200.166.145 10.0.0.24 <none> <none>

kubernetes-dashboard dashboard-metrics-scraper-c45b7869d-zdt8p 1/1 Running 0 53s 10.200.166.147 10.0.0.24 <none> <none>

kubernetes-dashboard kubernetes-dashboard-576cb95f94-wl8xb 1/1 Running 0 53s 10.200.166.148 10.0.0.24 <none> <none>

2.3创建用户

apiVersion: v1

kind: ServiceAccount

metadata:

namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRoleBinding

metadata:

roleRef:

apiGroup: rbac.authorization.k8s.io

kind: ClusterRole

subjects:

- kind: ServiceAccount

namespace: kubernetes-dashboard

root@master1:~/maniftest/dashboard# kubectl apply -f admin-user.yaml

serviceaccount/admin-user created

clusterrolebinding.rbac.authorization.k8s.io/admin-user created

root@master1:~/maniftest/dashboard# kubectl describe secret admin-user-token-b4c8n -n kubernetes-dashboard

Name: admin-user-token-b4c8n

Namespace: kubernetes-dashboard

Labels: <none>

Annotations: kubernetes.io/service-account.name: admin-user

kubernetes.io/service-account.uid: 2c3de8ea-a919-4b94-aa58-4a99aeef9e2e

Type: kubernetes.io/service-account-token

Data

====

ca.crt: 1350 bytes

namespace: 20 bytes

token: eyJhbGciOiJSUzI1NiIsImtpZCI6IkhWb0RMRkFCQUFTTm13Q1VzSFNzaVd3QW5fZjZOeW5QQklqSlBIclptUUEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWI0YzhuIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiIyYzNkZThlYS1hOTE5LTRiOTQtYWE1OC00YTk5YWVlZjllMmUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.RJuNGpH4ekbO73TOaB7v9w7RuL5ag1lfcgV0q3nq9u9l404Lqt2x4uOVH1uhQo3A6MdluKTYDC_l1xI2f1FKq-Q2G0a8e5XHFhaCOZKPfhG1-bqAYC9CRwvqmB3DUdGsHjznKINsfmcpkGZu4_V91h5DE_iyULT3ym6v466Bv37E_3P4D8aKXRaWWVWKARs7O1bMx_yp19aioAuQOMAYnF5ua7yJtZfFVUDwg81MyQ0C0qJIT6FhQmkTCEp2VbbOnWjoLUwEuqlWF1w_zMo0QJy83fTdrvY30hqvY1bQzs2lJ93UvEylPqdl0iVui2FgmLufJW7pZt45sAZwfMu_jw

2.4登录

三、kubectl常用命令

3.1kubectl get 类型名字 -n 名称空间查看资源

- o wide 显示更多

-w 动态显示

3.2kubectl describe 类型名字 -n 名称空间查看资源详细信息

3.3kubectl explain 类型查看资源配置选项

3.4 kubectl logs 容器查看容器的日志

3.5 kubectl create/run 创建资源对象

3.6 kubectl apply -f 文件名从配置文件创建对象

3.7 kubectl delete 删除对象

四、ETCD

4.1 etcdctl 管理命令

/usr/bin/etcdctl --write-out=table member list --endpoints=https://10.0.0.31:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem

+------------------+---------+----------------+------------------------+------------------------+------------+

+------------------+---------+----------------+------------------------+------------------------+------------+

+------------------+---------+----------------+------------------------+------------------------+------------+

root@etcd03:~# export NODE_IPS="10.0.0.29 10.0.0.30 10.0.0.31"

root@etcd03:~# for ip in ${NODE_IPS}; do /usr/bin/etcdctl --write-out=table endpoint status --endpoints=https://${ip}:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem; done

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

| https://10.0.0.29:2379 | 54e62bec6a0fc430 | 3.5.0 | 3.8 MB | false | false | 8 | 270378 | 270378 | |

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

| https://10.0.0.30:2379 | 730ef4b6cd5f7961 | 3.5.0 | 3.9 MB | false | false | 8 | 270378 | 270378 | |

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

| https://10.0.0.31:2379 | 56dc5cf57fee97f0 | 3.5.0 | 3.9 MB | true | false | 8 | 270378 | 270378 | |

+------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

root@etcd03:~# etcdctl get / --prefix --keys-only |more

/calico/ipam/v2/assignment/ipv4/block/10.200.104.0-26

/calico/ipam/v2/assignment/ipv4/block/10.200.166.128-26

/calico/ipam/v2/assignment/ipv4/block/10.200.180.0-26

/calico/ipam/v2/assignment/ipv4/block/10.200.199.64-26

/calico/ipam/v2/handle/ipip-tunnel-addr-master1.fchiaas.lcoal

/calico/ipam/v2/handle/ipip-tunnel-addr-master2

/calico/ipam/v2/handle/ipip-tunnel-addr-node1

/calico/ipam/v2/handle/ipip-tunnel-addr-node2

/calico/ipam/v2/handle/k8s-pod-network.05d2b3a59141b1292e2b766487486c72791e47bf23909fc5a02621b2689c6aa7

/calico/ipam/v2/handle/k8s-pod-network.0bc7559a48f40b946f3e47720539c21d544bee879fce8ac129a99a97a550d1c1

/calico/ipam/v2/handle/k8s-pod-network.15aaeff70b699297ab10df3673b25d2f4a0773f77016d47ea063f980ff09c570

/calico/ipam/v2/handle/k8s-pod-network.1bb56fbf532b66f3879772a7838bd38152773f64ad2b2124c7b01866902dfcbe

/calico/ipam/v2/handle/k8s-pod-network.1d28e1e9a173647705d05cb4d8329b8e852206172437193841dea37743046643

/calico/ipam/v2/handle/k8s-pod-network.3418cd370d8e070f1ddd948f9d8d3afdff5831ad8533611870b01bc19650e89b

/calico/ipam/v2/handle/k8s-pod-network.628b48a4cd51db026e7d0346e2c5336f05f8c85065c76ea7efcde8a264a74362

/calico/ipam/v2/handle/k8s-pod-network.679862357d2a47e326303d8aaf4ef621092d3d3a5e265db7f4d13bf097784210

/calico/ipam/v2/handle/k8s-pod-network.83309a03c182d93bf07ee228850f5089dd60c3b3e3e52847663628fa9df5fdda

/calico/ipam/v2/handle/k8s-pod-network.d52996649ccef1df19aafa4417caf5d70ac37afec20eed4e4066fca133405e80

/calico/ipam/v2/handle/k8s-pod-network.df5f0bcf96df77a8d778fb47f5a1d5ed4217b718a75252d39c64ceffbd930119

/calico/ipam/v2/host/master1.fchiaas.lcoal/ipv4/block/10.200.199.64-26

/calico/ipam/v2/host/master2/ipv4/block/10.200.180.0-26

/calico/ipam/v2/host/node1/ipv4/block/10.200.166.128-26

--More--

root@etcd03:~# etcdctl get /registry/pods/default/test

/registry/pods/default/test

k8s

v1Pod¥

testdefault"*$c2645c73-f9ce-4f6f-9280-8aba8c0c9d662𞅏Z

runtestz½

kubectl-runUpdatev𞅏FieldsV1:

{"f:metadata":{"f:labels":{".":{},"f:run":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"test\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}B󿾄

kubeletUpdatev¢¯FieldsV1:

{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.200.166.150\"}":{".":{},"f:ip":{}}},"f:startTime":{}}}Bstatus

kube-api-access-zqw5xkЁh

token

kube-root-ca.crt

ca.crtca.crt

namespace

v1metadata.namespace¤

testcentos"sleep"100000*BJL

kube-api-access-zqw5x-/var/run/secrets/kubernetes.io/serviceaccount"2j/dev/termination-logrAlways¢FileAlways 2

ClusterFirstBdefaultJdefaultR 10.0.0.24X`hrdefault-scheduler²6

node.kubernetes.io/not-readyExists" NoExecute(¬²8

node.kubernetes.io/unreachableExists" NoExecute(¬Ɓ񿂺PreemptLowerPriority¯

Running#

InitializedTru𞅏*2

ReadyTru¢¯*2'

ContainersReadyTru¢¯*2$

PodScheduledTru𞅏*2"* 10.0.0.24210.200.166.15𞅏Bł

test

¡¯centos:latest:`docker-pullable://centos@sha256:a27fd8080b517143cbbbab9dfb7c8571c40d67d534bbdee55bd6c473f432b177BIdocker://bb960f24afeb61e4311587d1293984656707e2c773626da0fca1296f4db860b5HJ

BestEffortZb

10.200.166.150"

root@etcd03:~# etcdctl put /data 1

root@etcd03:~# etcdctl put /data 2

root@etcd03:~# etcdctl del /data 2

341

root@etcd03:~# etcdctl watch /data

PUT

/data

PUT

/data

DELETE

/data

4.2数据备份和恢复

etcdctl snaphshot save 备份文件

root@etcd03:~# etcdctl snapshot save backup1

{"level":"info","ts":1642224623.165505,"caller":"snapshot/v3_snapshot.go:68","msg":"created temporary db file","path":"backup1.part"}

{"level":"info","ts":1642224623.1690366,"logger":"client","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"}

{"level":"info","ts":1642224623.1690714,"caller":"snapshot/v3_snapshot.go:76","msg":"fetching snapshot","endpoint":"127.0.0.1:2379"}

{"level":"info","ts":1642224623.2504776,"logger":"client","caller":"v3/maintenance.go:219","msg":"completed snapshot read; closing"}

{"level":"info","ts":1642224623.2565012,"caller":"snapshot/v3_snapshot.go:91","msg":"fetched snapshot","endpoint":"127.0.0.1:2379","size":"3.9 MB","took":"now"}

{"level":"info","ts":1642224623.2565756,"caller":"snapshot/v3_snapshot.go:100","msg":"saved","path":"backup1"}

Snapshot saved at backup1

root@etcd03:~# ls

backup1 snap

root@etcd03:~# ls

4.2数据恢复

etcdctl snapshot restore 备份名字 --data-dir=恢复的目录 #要求恢复目录内不能有东西

root@etcd03:~# etcdctl snapshot restore backup1 --data-dir=/opt/etcd

Deprecated: Use `etcdutl snapshot restore` instead.

2022-01-15T13:34:12+08:00 info snapshot/v3_snapshot.go:251 restoring snapshot {"path": "backup1", "wal-dir": "/opt/etcd/member/wal", "data-dir": "/opt/etcd", "snap-dir": "/opt/etcd/member/snap", "stack": "go.etcd.io/etcd/etcdutl/v3/snapshot.(*v3Manager).Restore\n\t/tmp/etcd-release-3.5.0/etcd/release/etcd/etcdutl/snapshot/v3_snapshot.go:257\ngo.etcd.io/etcd/etcdutl/v3/etcdutl.SnapshotRestoreCommandFunc\n\t/tmp/etcd-release-3.5.0/etcd/release/etcd/etcdutl/etcdutl/snapshot_command.go:147\ngo.etcd.io/etcd/etcdctl/v3/ctlv3/command.snapshotRestoreCommandFunc\n\t/tmp/etcd-release-3.5.0/etcd/release/etcd/etcdctl/ctlv3/command/snapshot_command.go:128\ngithub.com/spf13/cobra.(*Command).execute\n\t/home/remote/sbatsche/.gvm/pkgsets/go1.16.3/global/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:856\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/home/remote/sbatsche/.gvm/pkgsets/go1.16.3/global/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960\ngithub.com/spf13/cobra.(*Command).Execute\n\t/home/remote/sbatsche/.gvm/pkgsets/go1.16.3/global/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897\ngo.etcd.io/etcd/etcdctl/v3/ctlv3.Start\n\t/tmp/etcd-release-3.5.0/etcd/release/etcd/etcdctl/ctlv3/ctl.go:107\ngo.etcd.io/etcd/etcdctl/v3/ctlv3.MustStart\n\t/tmp/etcd-release-3.5.0/etcd/release/etcd/etcdctl/ctlv3/ctl.go:111\nmain.main\n\t/tmp/etcd-release-3.5.0/etcd/release/etcd/etcdctl/main.go:59\nruntime.main\n\t/home/remote/sbatsche/.gvm/gos/go1.16.3/src/runtime/proc.go:225"}

2022-01-15T13:34:12+08:00 info membership/store.go:119 Trimming membership information from the backend...

2022-01-15T13:34:12+08:00 info membership/cluster.go:393 added member {"cluster-id": "cdf818194e3a8c32", "local-member-id": "0", "added-peer-id": "8e9e05c52164694d", "added-peer-peer-urls": ["http://localhost:2380"]}

2022-01-15T13:34:12+08:00 info snapshot/v3_snapshot.go:272 restored snapshot {"path": "backup1", "wal-dir": "/opt/etcd/member/wal", "data-dir": "/opt/etcd", "snap-dir": "/opt/etcd/member/snap"}

root@etcd03:~# etcdctl snapshot restore backup1 --data-dir=/opt/etcd

Deprecated: Use `etcdutl snapshot restore` instead.

Error: data-dir "/opt/etcd" not empty or could not be read

root@etcd03:~#

五、增加master和node节点

5.1、现状

root@master1:~/manifests/coredns# kubectl get nodes

NAME STATUS ROLES AGE VERSION

10.0.0.21 Ready,SchedulingDisabled master 11m v1.22.2

10.0.0.22 Ready,SchedulingDisabled master 11m v1.22.2

10.0.0.24 Ready node 9m54s v1.22.2

10.0.0.25 Ready node 9m54s v1.22.2

5.2添加master和node

root@master1:/etc/kubeasz# ./ezctl add-master k8s-cluster1 10.0.0.23

root@master1:/etc/kubeasz# kubectl get nodes

NAME STATUS ROLES AGE VERSION

10.0.0.21 Ready,SchedulingDisabled master 21m v1.22.2

10.0.0.22 Ready,SchedulingDisabled master 21m v1.22.2

10.0.0.23 Ready,SchedulingDisabled master 114s v1.22.2

10.0.0.24 Ready node 20m v1.22.2

10.0.0.25 Ready node 20m v1.22.2

root@node2:/etc/kube-lb# cat conf/kube-lb.conf

user root;

worker_processes 1;

error_log /etc/kube-lb/logs/error.log warn;

events {

worker_connections 3000;

}

stream {

upstream backend {

server 10.0.0.23:6443 max_fails=2 fail_timeout=3s;

server 10.0.0.21:6443 max_fails=2 fail_timeout=3s;

server 10.0.0.22:6443 max_fails=2 fail_timeout=3s;

}

server {

listen 127.0.0.1:6443;

proxy_connect_timeout 1s;

proxy_pass backend;

}

root@master1:/etc/kubeasz# ./ezctl add-node k8s-cluster1 10.0.0.26

root@master1:/etc/kubeasz# kubectl get nodes

NAME STATUS ROLES AGE VERSION

10.0.0.21 Ready,SchedulingDisabled master 29m v1.22.2

10.0.0.22 Ready,SchedulingDisabled master 29m v1.22.2

10.0.0.23 Ready,SchedulingDisabled master 10m v1.22.2

10.0.0.24 Ready node 28m v1.22.2

10.0.0.25 Ready node 28m v1.22.2

10.0.0.26 Ready node 2m13s v1.22.2

5.3 升级

下载下面的文件（升级到的版本）

kubernetes-client-linux-amd64.tar.gz kubernetes-node-linux-amd64.tar.gz kubernetes-server-linux-amd64.tar.gz kubernetes.tar.gz

解压

升级时先升级master，然后升级node

升级master

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# systemctl stop kube-apiserver.service kube-controller-manager.service kubelet.service kube-proxy.service kube-scheduler.service

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# scp kube-apiserver kube-controller-manager kube-proxy kube-scheduler kubelet 10.0.0.21:/usr/bin/

kube-apiserver 100% 119MB 124.8MB/s 00:00

kube-controller-manager 100% 113MB 125.2MB/s 00:00

kube-proxy 100% 41MB 129.1MB/s 00:00

kube-scheduler 100% 47MB 81.9MB/s 00:00

kubelet 100% 116MB 121.7MB/s 00:00

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# systemctl start kube-apiserver.service kube-controller-manager.service kubelet.service kube-proxy.service kube-scheduler.service

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# kubectl get nodes

NAME STATUS ROLES AGE VERSION

10.0.0.21 Ready,SchedulingDisabled master 66m v1.22.5

10.0.0.22 Ready,SchedulingDisabled master 66m v1.22.2

10.0.0.23 Ready,SchedulingDisabled master 47m v1.22.2

10.0.0.24 Ready node 65m v1.22.2

10.0.0.25 Ready node 65m v1.22.2

10.0.0.26 Ready node 39m v1.22.2

相同的步骤升级剩余的master

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# kubectl get nodes

NAME STATUS ROLES AGE VERSION

10.0.0.21 Ready,SchedulingDisabled master 71m v1.22.5

10.0.0.22 Ready,SchedulingDisabled master 71m v1.22.5

10.0.0.23 Ready,SchedulingDisabled master 51m v1.22.5

10.0.0.24 Ready node 69m v1.22.2

10.0.0.25 Ready node 69m v1.22.2

10.0.0.26 Ready node 43m v1.22.2

然后升级node

root@node1:~# systemctl stop kubelet.service kube-proxy.service

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# scp kubectl kube-proxy kubelet 10.0.0.24:/usr/bin/

kubectl 100% 45MB 118.5MB/s 00:00

kube-proxy 100% 41MB 113.4MB/s 00:00

kubelet 100% 116MB 92.0MB/s 00:01

root@node1:~# systemctl start kubelet.service kube-proxy.service

安装上述步骤升级其余节点

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# kubectl get nodes

NAME STATUS ROLES AGE VERSION

10.0.0.21 Ready,SchedulingDisabled master 79m v1.22.5

10.0.0.22 Ready,SchedulingDisabled master 79m v1.22.5

10.0.0.23 Ready,SchedulingDisabled master 59m v1.22.5

10.0.0.24 Ready node 78m v1.22.5

10.0.0.25 Ready node 78m v1.22.5

10.0.0.26 Ready node 51m v1.22.5

把部署用的原始文件替换掉

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# cp kube-apiserver kube-controller-manager kube-proxy kube-scheduler kubelet kubectl /etc/kubeasz/bin/

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# ll /etc/kubeasz/bin/

total 878000

drwxr-xr-x 3 root root 4096 Jan 3 08:10 ./

drwxrwxr-x 12 root root 4096 Jan 15 17:19 ../

-rwxr-xr-x 1 root root 4536104 Feb 5 2021 bridge*

-rwxr-xr-x 1 root root 44978176 Aug 24 22:13 calicoctl*

-rwxr-xr-x 1 root root 10376657 Aug 24 22:12 cfssl*

-rwxr-xr-x 1 root root 6595195 Aug 24 22:13 cfssl-certinfo*

-rwxr-xr-x 1 root root 2277873 Aug 24 22:12 cfssljson*

-rwxr-xr-x 1 root root 1046424 Aug 24 22:12 chronyd*

-rwxr-xr-x 1 root root 36789288 Jan 3 08:09 containerd*

drwxr-xr-x 2 root root 4096 Aug 24 22:13 containerd-bin/

-rwxr-xr-x 1 root root 7172096 Jan 3 08:09 containerd-shim*

-rwxr-xr-x 1 root root 19161064 Jan 3 08:09 ctr*

-rwxr-xr-x 1 root root 61133792 Jan 3 08:09 docker*

-rwxr-xr-x 1 root root 11748168 Aug 24 22:13 docker-compose*

-rwxr-xr-x 1 root root 71555008 Jan 3 08:09 dockerd*

-rwxr-xr-x 1 root root 708616 Jan 3 08:09 docker-init*

-rwxr-xr-x 1 root root 2928566 Jan 3 08:09 docker-proxy*

-rwxr-xr-x 1 root root 23560192 Jun 16 2021 etcd*

-rwxr-xr-x 1 root root 17969152 Jun 16 2021 etcdctl*

-rwxr-xr-x 1 root root 3357992 Feb 5 2021 flannel*

-rwxr-xr-x 1 root root 41603072 Dec 9 2020 helm*

-rwxr-xr-x 1 root root 3565330 Feb 5 2021 host-local*

-rwxr-xr-x 1 root root 1305408 Aug 24 22:12 keepalived*

-rwxr-xr-x 1 root root 124809216 Jan 15 18:52 kube-apiserver*

-rwxr-xr-x 1 root root 118497280 Jan 15 18:52 kube-controller-manager*

-rwxr-xr-x 1 root root 46936064 Jan 15 18:52 kubectl*

-rwxr-xr-x 1 root root 121213080 Jan 15 18:52 kubelet*

-rwxr-xr-x 1 root root 43479040 Jan 15 18:52 kube-proxy*

-rwxr-xr-x 1 root root 49168384 Jan 15 18:52 kube-scheduler*

-rwxr-xr-x 1 root root 3530531 Feb 5 2021 loopback*

-rwxr-xr-x 1 root root 1777808 Aug 24 22:12 nginx*

-rwxr-xr-x 1 root root 3966455 Feb 5 2021 portmap*

-rwxr-xr-x 1 root root 9600824 Jan 3 08:09 runc*

-rwxr-xr-x 1 root root 3668289 Feb 5 2021 tuning*

root@master1:/opt/soft/kubernetes/kubernetes/server/bin# /etc/kubeasz/bin/kube-apiserver --version

六、基于yaml文件创建nginx及tomcat并实现动静分离

1.1 ubuntu的Dockerfile

Xshellroot@cncf-docker:/opt/dockerfile/system/ubuntu/20.04# cat Dockerfile

FROM ubuntu:20.04

maintainer "dongxikui"

RUN apt update && apt -y install vim wget tree lrzsz gcc build-essential automake libpcre3 libpcre3-dev zlib1g-dev zlib1g openssl libssl-dev iproute2 net-tools iotop make curl &&useradd nginx -u 2001 && groupadd www -g 2022 && useradd www -u 2022 -g www

ADD Shanghai /usr/share/zoneinfo/Asia/

RUN rm -fr /etc/localtime && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

1.2 nginx的Dockerfile和配置文件

root@cncf-docker:/opt/dockerfile/web/nginx# cat Dockerfile

FROM reg.fchiaas.local/cncf/ubuntu-20-04:v1

maintainer "dongxikui"

ADD nginx-1.18.0.tar.gz /usr/local/src/

RUN cd /usr/local/src/nginx-1.18.0 && ./configure --prefix=/apps/nginx --with-http_sub_module && make && make install

ADD nginx.conf /apps/nginx/conf/nginx.conf

ADD code.tar.gz /data/nginx/html/webapp

ADD run_nginx.sh /apps/nginx/sbin/

RUN mkdir -p /data/nginx/html/webapp/static /data/nginx/html/webapp/images && chmod a+x /apps/nginx/sbin/run_nginx.sh && chown -R nginx.nginx /apps/* /data/*

EXPOSE 80 443

CMD ["/apps/nginx/sbin/run_nginx.sh"]

root@cncf-docker:/opt/dockerfile/web/nginx# cat nginx.conf

user nginx nginx;

worker_processes auto;

#error_log logs/error.log;

#error_log logs/error.log notice;

#error_log logs/error.log info;

#pid logs/nginx.pid;

daemon off;

events {

worker_connections 1024;

}

http {

include mime.types;

default_type application/octet-stream;

#log_format main '$remote_addr - $remote_user [$time_local] "$request" '

# '$status $body_bytes_sent "$http_referer" '

# '"$http_user_agent" "$http_x_forwarded_for"';

#access_log logs/access.log main;

sendfile on;

#tcp_nopush on;

#keepalive_timeout 0;

keepalive_timeout 65;

#gzip on;

upstream tomcat_webserver {

server magedu-tomcat-app1-service.magedu.svc.fchiaas.local:80;

}

server {

listen 80;

server_name localhost;

#charset koi8-r;

#access_log logs/host.access.log main;

location / {

root /data/nginx/html/;

index index.html index.htm;

}

location /data/nginx/html/webapp {

root html;

index index.html index.htm;

}

location /myapp {

proxy_pass http://tomcat_webserver;

proxy_set_header Host $host;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header X-Real-IP $remote_addr;

}

#error_page 404 /404.html;

# redirect server error pages to the static page /50x.html

error_page 500 502 503 504 /50x.html;

location = /50x.html {

root html;

}

# proxy the PHP scripts to Apache listening on 127.0.0.1:80

#location ~ \.php$ {

# proxy_pass http://127.0.0.1;

# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000

#location ~ \.php$ {

# root html;

# fastcgi_pass 127.0.0.1:9000;

# fastcgi_index index.php;

# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;

# include fastcgi_params;

# deny access to .htaccess files, if Apache's document root

# concurs with nginx's one

#location ~ /\.ht {

# deny all;

}

# another virtual host using mix of IP-, name-, and port-based configuration

#server {

# listen 8000;

# listen somename:8080;

# server_name somename alias another.alias;

# location / {

# root html;

# index index.html index.htm;

# }

# HTTPS server

#server {

# listen 443 ssl;

# server_name localhost;

# ssl_certificate cert.pem;

# ssl_certificate_key cert.key;

# ssl_session_cache shared:SSL:1m;

# ssl_session_timeout 5m;

# ssl_ciphers HIGH:!aNULL:!MD5;

# ssl_prefer_server_ciphers on;

# location / {

# root html;

# index index.html index.htm;

# }

}

1.3JDK的Dockerfile

root@cncf-docker:/opt/dockerfile/web/jdk/jdk1.18.0# cat Dockerfile

。。。。。。

unset i

unset -f pathmunge

添加的部分

export JAVA_HOME=/usr/local/jdk

export TOMCAT_HOME=/apps/tomcat

export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$TOMCAT_HOME/bin:$PATH

export CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar

1.4tomcat的Dockerfile

root@cncf-docker:/opt/dockerfile/web/tomcat/tomcat-ubuntu# cat Dockerfile

FROM reg.fchiaas.local/cncf/ubuntu-2004-jdk-1_8_241:v1

LABEL maintainer="dongxikui"

ADD apache-tomcat-8.5.65.tar.gz /apps/

RUN ln -sv /apps/apache-tomcat-8.5.65 /apps/tomcat

ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh

RUN chmod a+x /apps/tomcat/bin/run_tomcat.sh

ADD myapp/* /data/tomcat/webapps/myapp/

ADD server.xml /apps/tomcat/conf/server.xml

RUN chown -R www.www /apps/* /data/*

EXPOSE 8080 8009

CMD ["/apps/tomcat/bin/run_tomcat.sh"]

root@cncf-docker:/opt/dockerfile/web/tomcat/tomcat-ubuntu# cat myapp/index.html

Test myapp1 tomcat page

root@cncf-docker:/opt/dockerfile/web/tomcat/tomcat-ubuntu# cat server.xml

下面是修改的部分

1.5nginx的yaml文件

root@master1:~/manifests/case/dynamic_static# cat nginx.yaml

apiVersion: apps/v1

kind: Deployment

metadata:

labels:

app: magedu-nginx-deployment-label

namespace: magedu

spec:

replicas: 1

selector:

matchLabels:

app: magedu-nginx-selector

template:

metadata:

labels:

app: magedu-nginx-selector

spec:

containers:

- name: magedu-nginx-container

image: reg.fchiaas.local/cncf/ubuntu-2004-nginx-1_18_0:v1

imagePullPolicy: Always

ports:

- containerPort: 80

protocol: TCP

- containerPort: 443

protocol: TCP

env:

- name: "password"

value: "123456"

- name: "age"

value: "20"

resources:

limits:

cpu: 1

memory: 2Gi

requests:

cpu: 500m

memory: 1Gi

volumeMounts:

- name: magedu-images

mountPath: /data/nginx/html/webapp/images

readOnly: false

- name: magedu-static

mountPath: /data/nginx/html/webapp/static

readOnly: false

volumes:

- name: magedu-images

nfs:

server: 10.0.0.20

path: /nfs/k8sdata/magedu/images

- name: magedu-static

nfs:

server: 10.0.0.20

path: /nfs/k8sdata/magedu/static

---

kind: Service

apiVersion: v1

metadata:

labels:

app: magedu-nginx-service-label

namespace: magedu

spec:

type: NodePort

selector:

app: magedu-nginx-selector

ports:

- name: http

port: 80

protocol: TCP

targetPort: 80

nodePort: 30090

- name: https

port: 443

protocol: TCP

targetPort: 443

nodePort: 30091

1.6tomcat的yaml文件

root@master1:~/manifests/case/dynamic_static# cat tomcat.yaml

apiVersion: apps/v1

kind: Deployment

metadata:

labels:

app: magedu-tomcat-app1-deployment-label

namespace: magedu

spec:

replicas: 1

selector:

matchLabels:

app: magedu-tomcat-app1-selector

template:

metadata:

labels:

app: magedu-tomcat-app1-selector

spec:

containers:

- name: magedu-tomcat-app1-container

image: reg.fchiaas.local/cncf/ubuntu-2004-jdk-1_8_241-tomcat-8_5_65:v1

imagePullPolicy: Always

ports:

- containerPort: 8080

protocol: TCP

env:

- name: "password"

value: "123456"

- name: "age"

value: "20"

resources:

limits:

cpu: 1

memory: 512Mi

requests:

cpu: 0.5

memory: 512Mi

# volumeMounts:

# - name: magedu-images

# mountPath: /data/nginx/html/webapp/images

# readOnly: false

# - name: magedu-static

# mountPath: /data/nginx/html/webapp/static

# readOnly: false

# volumes:

# - name: magedu-images

# server: 10.0.0.20

# path: /nfs/k8sdata/magedu/images

# - name: magedu-static

# server: 10.0.0.20

# path: /nfs/k8sdata/magedu/static

# nodeSelector:

# project: magedu

# app: tomcat

---

kind: Service

apiVersion: v1

metadata:

labels:

app: magedu-tomcat-app1-service-label

namespace: magedu

spec:

# type: NodePort

ports:

- name: http

port: 80

protocol: TCP

targetPort: 8080

# nodePort: 30092

selector:

app: magedu-tomcat-app1-selector

1.7 测试

root@master1:~/manifests/case/dynamic_static# kubectl apply -f tomcat.yaml

deployment.apps/magedu-tomcat-app1-deployment created

service/magedu-tomcat-app1-service created

root@master1:~/manifests/case/dynamic_static# kubectl apply -f nginx.yaml

deployment.apps/mangedu-nginx-deployment created

service/magedu-nginx-service created

root@master1:~/manifests/case/dynamic_static# kubectl get pod -A

NAMESPACE NAME READY STATUS RESTARTS AGE

kube-system calico-kube-controllers-59df8b6856-44lmg 1/1 Running 8 (3h30m ago) 10d

kube-system calico-node-4sk4x 1/1 Running 9 (3h31m ago) 11d

kube-system calico-node-7hkv2 1/1 Running 9 (3h30m ago) 11d

kube-system calico-node-7mh6m 1/1 Running 9 (3h31m ago) 11d

kube-system calico-node-qqndf 1/1 Running 10 (3h30m ago) 11d

kube-system calico-node-szrbs 1/1 Running 9 (3h30m ago) 11d

kube-system calico-node-xrdlz 1/1 Running 11 (46h ago) 11d

kube-system coredns-69d84cdc49-d2lcq 1/1 Running 8 (3h30m ago) 10d

kube-system coredns-69d84cdc49-ztgv4 1/1 Running 10 (3h30m ago) 11d

kubernetes-dashboard dashboard-metrics-scraper-c45b7869d-rnm7t 1/1 Running 7 (3h30m ago) 7d22h

kubernetes-dashboard kubernetes-dashboard-576cb95f94-flnsj 1/1 Running 6 (3h30m ago) 6d3h

magedu magedu-tomcat-app1-deployment-5dcbbd96b4-htfwg 1/1 Running 0 15s

magedu mangedu-nginx-deployment-775c8d479b-mxwxt 1/1 Running 0 5s

root@master1:~/manifests/case/dynamic_static# kubectl get svc -A

NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE

default kubernetes ClusterIP 10.100.0.1 <none> 443/TCP 11d

default service-nfs-probe NodePort 10.100.130.39 <none> 81:30081/TCP 4d5h

kube-system kube-dns ClusterIP 10.100.0.2 <none> 53/UDP,53/TCP,9153/TCP 11d

kubernetes-dashboard dashboard-metrics-scraper ClusterIP 10.100.21.70 <none> 8000/TCP 7d22h

kubernetes-dashboard kubernetes-dashboard NodePort 10.100.89.237 <none> 443:30443/TCP 7d22h

magedu magedu-nginx-service NodePort 10.100.205.204 <none> 80:30090/TCP,443:30091/TCP 36s

magedu magedu-tomcat-app1-service ClusterIP 10.100.120.246 <none> 80/TCP 46s

人面猴
序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
沈念sama阅读 205,386评论 6赞 479
死咒
序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
沈念sama阅读 87,939评论 2赞 381
救了他两次的神仙让他今天三更去死
文/潘晓璐我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
开封第一讲书人阅读 151,851评论 0赞 341
道士缉凶录：失踪的卖姜人
文/不坏的土叔我叫张陵，是天一观的道长。经常有香客问我，道长，这世上最难降的妖魔是什么？我笑而不...
开封第一讲书人阅读 54,953评论 1赞 278
港岛之恋（遗憾婚礼）
正文为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
茶点故事阅读 63,971评论 5赞 369
恶毒庶女顶嫁案：这布局不是一般人想出来的
文/花漫我一把揭开白布。她就那样静静地躺着，像睡着了一般。火红的嫁衣衬着肌肤如雪。梳的纹丝不乱的头发上，一...
开封第一讲书人阅读 48,784评论 1赞 283
城市分裂传说
那天，我揣着相机与录音，去河边找鬼。笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。我是一名探鬼主播，决...
沈念sama阅读 38,126评论 3赞 399
双鸳鸯连环套：你想象不到人心有多黑
文/苍兰香墨我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
开封第一讲书人阅读 36,765评论 0赞 258
万荣杀人案实录
序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
沈念sama阅读 43,148评论 1赞 300
护林员之死
正文独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋以下内容为张勋视角年9月15日...
茶点故事阅读 35,744评论 2赞 323
白月光启示录
正文我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
茶点故事阅读 37,858评论 1赞 333
活死人
序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
沈念sama阅读 33,479评论 4赞 322
日本核电站爆炸内幕
正文年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
茶点故事阅读 39,080评论 3赞 307
男人毒药：我在死后第九天来索命
文/蒙蒙一、第九天我趴在偏房一处隐蔽的房顶上张望。院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
开封第一讲书人阅读 30,053评论 0赞 19
一桩弑父案，背后竟有这般阴谋
文/苍兰香墨我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。一阵脚步声响...
开封第一讲书人阅读 31,278评论 1赞 260
情欲美人皮
我被黑心中介骗来泰国打工，没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。一个月前我还...
沈念sama阅读 45,245评论 2赞 352
代替公主和亲
正文我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
茶点故事阅读 42,590评论 2赞 343

云原生第三课作业

推荐阅读更多精彩内容