nginx-log.conf
input {
file {
path =>"/nginx/access.log"
start_position => "beginning" #第一次从头收集，之后从新添加的日志收集
stat_interval =>"3" #日志收集的间隔时间
type=>"nginx_access"
sincedb_path => [ "/tmp/nginx_access-01.data" ]
}
file {
path =>"/nginx/error.log"
start_position => "beginning" #第一次从头收集，之后从新添加的日志收集
stat_interval =>"3" #日志收集的间隔时间
type=>"nginx_error"
sincedb_path => [ "/tmp/nginx_error_11b.data" ]
codec => multiline{
pattern => "^\d{4}\/"
what => "previous"
negate => true
auto_flush_interval => 1
}
}
}
filter {
if [type] == "nginx_access" {
grok {
match => { "message" => "%{IPORHOST:remote_addr} - %{DATA:remote_user} \[%{HTTPDATE:time_local}\] \"%{WORD:request_method} %{DATA:uri} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent_bytes} \"%{DATA:http_referrer}\" \"%{DATA:http_user_agent}\"" }
}
# 通过date插件，把nginx日志中的时间戳用作logstash的event时间戳
date {
match => [ "time_local", "dd/MMM/yyyy:HH:mm:ss Z" ]
}
} else if [type] == "nginx_error"{
grok {
match => { "message" => "(?<time_local>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:log_level}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:error_message}(?:, client: (?<clientip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)?(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:http_referrer}\")?" }
}
# 通过date插件，把nginx日志中的时间戳用作logstash的event时间戳
date {
match => [ "time_local", "YYYY/MM/dd HH:mm:ss" ]
target => "@timestamp"
}
mutate{
remove_field => "time_local"
}
}
}
output {
if [type] == "nginx_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "nginx-access-%{+YYYY.MM.dd}"
user => "logstash_write"
password=>"logstash_write"
}
}
if [type] == "nginx_error" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "nginx-error-%{+YYYY.MM.dd}"
user => "logstash_write"
password=>"logstash_write"
}
}
}
