Kubernetes ELLK
本次方案是按照 Elasticsearch + Logspout + Logstash + kibana 并且手机的日志可以被简单切分：
Elasticsearch-rc 配置文件：
apiVersion: v1
kind: ReplicationController
metadata:
name: elasticsearch-logging-v1
labels:
k8s-app: elasticsearch-logging
version: v1
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
k8s-app: elasticsearch-logging
version: v1
template:
metadata:
labels:
k8s-app: elasticsearch-logging
version: v1
kubernetes.io/cluster-service: "true"
spec:
nodeSelector:
role: elk
containers:
- image: registry.aliyuncs.com/slzcc/elasticsearch
name: elasticsearch
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
volumeMounts:
- name: es-persistent-storage
mountPath: "/usr/share/elasticsearch/data"
volumes:
- name: es-persistent-storage
hostPath:
path: "/data/elasticsearch"
Elasticsearch-svc 配置文件：
apiVersion: v1
kind: Service
metadata:
name: elasticsearch-logging
labels:
k8s-app: elasticsearch-logging
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Elasticsearch"
spec:
ports:
- port: 9200
name: http
protocol: TCP
targetPort: db
selector:
k8s-app: elasticsearch-logging
Kibana-rc 配置文件：
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kibana-logging
labels:
k8s-app: kibana-logging
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kibana-logging
template:
metadata:
labels:
k8s-app: kibana-logging
spec:
nodeSelector:
role: elk
containers:
- name: kibana-logging
image: registry.aliyuncs.com/slzcc/kibana
resources:
# keep request = limit to keep this container in guaranteed class
limits:
cpu: 100m
requests:
cpu: 100m
env:
- name: "ELASTICSEARCH_URL"
value: "http://elasticsearch-logging:9200"
ports:
- containerPort: 5601
name: ui
protocol: TCP
Kibana-svc 配置文件：
apiVersion: v1
kind: Service
metadata:
name: kibana-logging
labels:
k8s-app: kibana-logging
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Kibana"
spec:
ports:
- port: 5601
name: http
protocol: TCP
targetPort: ui
selector:
k8s-app: kibana-logging
Logstash-configmap 配置文件：
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash
data:
logstash.conf: |-
input {
udp {
port => 514
type => syslog
codec => json
}
tcp {
port => 514
type => syslog
codec => json
}
}
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOG5424PRI}%{NONNEGINT:ver} +(?:%{TIMESTAMP_ISO8601:ts}|-) +(?:%{HOSTNAME:containerid}|-) +(?:% {NOTSPACE:containername}|-) +(?:%{NOTSPACE:proc}|-) +(?:%{WORD:msgid}|-) +(?:%{SYSLOG5424SD:sd}|-|) +%{GREEDYDATA:msg}" }
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "@source_host", "%{syslog_hostname}" ]
replace => [ "@message", "%{syslog_message}" ]
}
}
mutate {
remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
}
}
}
output {
elasticsearch {
hosts => ["elasticsearch-logging:9200"]
index => "k8s-%{type}-%{+YYYY.MM.dd}"
document_type => "%{type}"
workers => 1
flush_size => 20000
idle_flush_time => 10
template_overwrite => true
codec => json
}
}
Logstash-rc 配置文件：
apiVersion: v1
kind: ReplicationController
metadata:
name: logstash
labels:
k8s-app: logstash
version: v1
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
k8s-app: logstash
version: v1
template:
metadata:
labels:
k8s-app: logstash
version: v1
kubernetes.io/cluster-service: "true"
spec:
nodeSelector:
role: elk
containers:
- image: registry.aliyuncs.com/slzcc/logstash-build
name: logstash
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 514
name: input
protocol: TCP
- containerPort: 514
name: output
protocol: UDP
command:
- '/logstash-5.1.1/bin/logstash'
- '-f'
- '/etc/logstash/logstash.conf'
- '-w 20'
volumeMounts:
- mountPath: "/etc/logstash/"
name: config-volume
volumes:
- name: config-volume
configMap:
name: logstash
Logstash-svc 配置文件：
apiVersion: v1
kind: Service
metadata:
name: logstash
labels:
k8s-app: logstash
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "logstash"
spec:
ports:
- port: 514
name: input
protocol: TCP
targetPort: input
# - port: 514
# name: output
# protocol: UDP
# targetPort: output
selector:
k8s-app: logstash
clusterIP: None
Lospout-daemon 配置文件：
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: logspout-elasticsearch
labels:
k8s-app: logspout-logging
spec:
template:
metadata:
labels:
name: logspout-elasticsearch
spec:
containers:
nodeSelector:
role: elk
- name: logspout-elasticsearch
image: registry.aliyuncs.com/slzcc/logspout-logstash
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 200Mi
env:
- name: "ROUTE_URIS"
value: "logstash+tcp://logstash:514"
volumeMounts:
- mountPath: "/var/run/docker.sock"
name: sock
volumes:
- hostPath:
path: "/var/run/docker.sock"
name: sock
terminationGracePeriodSeconds: 30
