image.png

Logstash Input

[root@logstash-node1 conf.d]# cat
input_file_output_console.conf
input {
    file {
        path => "/var/log/oldxu.log"
        type => syslog
exclude => "*.gz" #不想监听的文件规则，基于glob匹配语法
        start_position => "beginning" #第一次丛头开始读取文件 beginning or end
        stat_interval => "3" #定时检查文件是否更新，默认1s
    }
}
output {
    stdout {
        codec => rubydebug
    }
}
[root@logstash-node1 conf.d]# cat
input_stdin_output_console.conf
input {
      stdin {
          type => stdin
          tags => "tags_stdin"
      }
}
output {
    stdout {
        codec => "rubydebug"
    }
}

logstash 分析Nginx

66.249.73.135 - - [20/May/2015:21:05:11 +0000] "GET
/blog/tags/xsendevent HTTP/1.1" 200 10049 "-"
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X)
AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0
Mobile/10A5376e Safari/8536.25 (compatible;
Googlebot/2.1; +http://www.google.com/bot.html)"

[root@logstash-node1 conf.d]# cat
input_filebeat_output_es.conf
input {
    beats {
        port => 5044
    }
}
filter {
if "nginx-access" in [tags][0] {
    grok {
        match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer}%{QS:useragent}" }
    }
    date {
        match => ["timestamp", "dd/MMM/yyyy:HH:mm:ssZ"]
        target => "@timestamp"
        timezone => "Asia/Shanghai"
    }
    geoip {
        source => "clientip"
    }
    useragent {
        source => "useragent"
        => "useragent"
    }
    mutate {
        rename => ["%{[host][name]}" ,"hostname" ]
        convert => [ "bytes", "integer" ]
        remove_field => [ "message", "agent" ,
"input","ecs" ]
        add_field => { "target_index" => "logstashnginx-access-%{+YYYY.MM.dd}" }
    }
} else if "nginx-error" in [tags][0] {
    mutate {
        add_field => { "target_index" => "logstashnginx-error-%{+YYYY.MM.dd}" }
    }
  }
}
output {
    elasticsearch {
        hosts =>
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
        index => "%{[target_index]}"
    }
}

Logstash 分析 MySQL

filebeat

[root@web01 filebeat]# cat filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/mariadb/slow.log
  exclude_lines: ['^\# Time']
  multiline.pattern: '^\# User'
  multiline.negate: true
  multiline.match: after
  multiline.max_lines: 10000
  tags: ["mysql-slow"]

output.logstash:
  hosts: ["10.0.0.151:5044"]

logstash

[root@logstash-node1 conf.d]# cat
input_filebeat_mysql_output_es.conf
input {
    beats {
        port => 5044
    }
}

filter {
    mutate {
        gsub => ["message","\n"," "]
    }
    grok {
        match => {
"message" => "(?m)^# User@Host: %{USER:User}\[%{USER-2:User}\] @ (?:(?<Clienthost>\S*) )?\[(?:%{IP:Client_IP})?\] # Thread_id: %{NUMBER:Thread_id:integer}\s+ Schema: (?:(?<DBname>\S*))\s+QC_hit: (?:(?<QC_hit>\S*) )# Query_time: %{NUMBER:Query_Time}\s+ Lock_time: %{NUMBER:Lock_Time}\s+ Rows_sent: %{NUMBER:Rows_Sent:integer}\s+Rows_examined: %{NUMBER:Rows_Examined:integer} SET timestamp=%{NUMBER:timestamp}; \s*(?<Query>(?<Action>\w+)\s+.*)"
    }
  }
  date {
      match => ["timestamp","UNIX", "YYYY-MM-ddHH:mm:ss"]
      target => "@timestamp"
      timezone => "Asia/Shanghai"
  }
  mutate {
      remove_field =>
["message","input","timestamp","agent","ecs","log"]
      convert => ["Lock_Time","float"]
      convert => ["Query_Time","float"]
      add_field => { "target_index" => "logstashmysql-slow-%{+YYYY.MM.dd}" }
  }
}
output {
    elasticsearch {
        hosts => ["10.0.0.161:9200"]
        index => "%{[target_index]}"
    }
    stdout {
        codec => "rubydebug"
    }
}

Logstash 分析 APP

模拟产生日志

java -jar app-dashboard-1.0-SNAPSHOT.jar&>/var/log/app.log

filebeat

filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/app.log
output.logstash:
hosts: ["10.0.0.151:5044"]

logstash

[root@logstash-node1 conf.d]# cat
input_filebeat_app_output_es.conf
input {
    beats {
        port => 5044
    }
}

filter {
    mutate {
        split => {"message" => "|"}
        add_field => {
            "UserID" => "%{[message][1]}"
            "Action" => "%{[message][2]}"
            "Date" => "%{[message][3]}"
        }
                convert => {
                        "UserID" => "integer"
                        "Action" => "string"
                        "Date" => "string"
                }
        }

        #2020-01-15 17:04:15
        date {
            match => ["Date","yyyy-MM-dd HH:mm:ss"]
            target => "@timestamp"
            timezone => "Asia/Chongqing"
        }

        mutate {
            #remove_field => ["message","Date"]
            add_field => { "target_index" => "logstash-app-%{+YYYY.MM.dd}" }
        }
}
output {
        elasticsearch {
            hosts => ["10.0.0.161:9200"]
            index => "%{[target_index]}"
            template_overwrite => true
        }
}