DBA课程-day20-ELK第二天

ELK第一天知识回顾

1.ELK各代表什么软件
2.传统方式收集日志的弊端
3.ELK收集日志的架构
4.日志流转流程
5.安装部署filebeat
- input   -->  nginx日志 
- output  -->  ES 
- ES      -->  kibana
6.收集普通nginx日志有哪些不足
- 日志字段内容不能拆分，不能单独显示
7.把nginx日志格式修改为json格式
8.filebeat配置文件添加2行json解析的参数
9.删除旧索引，重启filebeat
10.在kibana里添加索引，测试数据
- 时间范围 鼠标选择区间  quick time 

ELK第二天内容

现在的架构的不合理的地方
1.按天生成太费事，不能聚合搜索某几天的数据
2.filebeat默认创建的索引不好区分是什么业务，不能区分域名

合理的规划
1.索引按月生成
2.按照域名

大家的索引命令方案

#业务名称_10.0.0.51_月份
#域名-date-ip地址
按照  域名-服务-年月  生成索引
filebeat-2019.07-www
#www.etiantian.org-nginx-2019.07
#www.etiantian.org-2019.07
#www.etiantian.org-2019.07
#nginx_www.etianrtian_2019.07
Nginx_www_2019-07

大家开会讨论筛选之后合理的索引命名

nginx_bbs_access_2019.9
nginx_www_access_2019.9
nginx_blog_access_2019.9

为什么filebeat默认索引有版本号

filebeat-6.6.0-2019.07.11

修改filebeat索引后报错记录

2019-07-11T09:30:05.062+0800    ERROR   instance/beat.go:911    
Exiting: setup.template.name and setup.template.pattern have to be set if index name is modified

提示我们需要添加2行配置
setup.template.name
setup.template.pattern

1.修改后filebeat自定义nginx日志索引配置

filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  index: "nginx-www-access-%{[beat.version]}-%{+yyyy.MM}"
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

2.nginx修改多域名多日志

[root@db01 /etc/nginx]# egrep -v "#|^$" /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
   log_format   json '{ "time_local": "$time_local", '
                           '"remote_addr": "$remote_addr", '
                           '"referer": "$http_referer", '
                           '"request": "$request", '
                           '"status": $status, '
                           '"bytes": $body_bytes_sent, '
                           '"agent": "$http_user_agent", '
                           '"x_forwarded": "$http_x_forwarded_for", '
                           '"up_addr": "$upstream_addr",'
                           '"up_host": "$upstream_http_host",'
                           '"upstream_time": "$upstream_response_time",'
                           '"request_time": "$request_time"'
    ' }';
    access_log  /var/log/nginx/access.log  json;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;
    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
    include /etc/nginx/conf.d/*.conf;
}

[root@db01 /etc/nginx/conf.d]# cat www.conf 
server {
    listen 80;
    server_name  www.oldboy.com;
    location / {
        root /html/www;
        index index.html;
    }
    access_log  /var/log/nginx/www_access.log  json;
}

[root@db01 /etc/nginx/conf.d]# cat bbs.conf 
server {
    listen 80;
    server_name  bbs.oldboy.com;
    location / {
        root /html/bbs;
        index index.html;
    }
    access_log  /var/log/nginx/bbs_access.log  json;
}

[root@db01 /etc/nginx/conf.d]# cat blog.conf 
server {
    listen 80;
    server_name  blog.oldboy.com;
    location / {
        root /html/blog;
        index index.html;
    }
    access_log  /var/log/nginx/blog_access.log  json;
}

mkdir /html/{www,blog,bbs} -p
echo "db01 www" > /html/www/index.html
echo "db01 bbs" > /html/bbs/index.html
echo "db01 blog" > /html/blog/index.html

chown -R nginx:nginx /html/
nginx -t
systemctl restart nginx