Logstash采集Nginx日志写入ES

Logstash采集Nginx日志方式

RPM安装Logstash采集Nginx日志
Docker安装Logstash采集Nginx日志

RPM安装Logstash采集Nginx日志

安装jdk

#新建jdk目录 ~/software/java
cd ~/software/java
#下载jdk1.8
wget http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz?AuthParam=1534129356_6b3ac55c6a38ba5a54c912855deb6a22
#解压
tar -zxvf jdk-8u181-linux-x64.tar.gz
#配置JAVA环境变量
vi /etc/profile
#写入以下内容
#java
export JAVA_HOME=/root/software/java/jdk1.8.0_181
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib
#加载环境变量
source /etc/profile

验证jdk

image.png

安装Logstash

#下载logstash rpm文件【与ES、Kibana版本一致】
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.8.0.rpm
#rpm安装
rpm -ivh logstash-7.8.0.rpm

etc/logstash/pipelines.yml 为管道配置，添加新的logstash配置文件后要检查管道中是否关联启用

Nginx配置文件

  #user  nobody;
  worker_processes  2;

  #error_log  logs/error.log;
  #error_log  logs/error.log  notice;
  #error_log  logs/error.log  info;

  #pid        logs/nginx.pid;

  events {
      worker_connections  1024;
  }

  http {
      include       mime.types;
      default_type  application/octet-stream;

      log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for" "$upstream_response_time" "$request_time"';

      #access log json格式配置
      log_format json '{ "@timestamp": "$time_iso8601", '
                     '"time": "$time_iso8601", '
                     '"remote_addr": "$remote_addr", '
                     '"remote_user": "$remote_user", '
                     '"body_bytes_sent": "$body_bytes_sent", '
                     '"request_time": "$request_time", '
                     '"status": "$status", '
                     '"host": "$host", '
                     '"request": "$request", '
                     '"request_method": "$request_method", '
                     '"uri": "$uri", '
                     '"http_referrer": "$http_referer", '
                     '"body_bytes_sent":"$body_bytes_sent", '
                     '"http_x_forwarded_for": "$http_x_forwarded_for", '
                     '"http_user_agent": "$http_user_agent" '
                '}';

      access_log  /var/log/nginx/access.log  main;
      #设置json格式access log文件路径
      access_log  /var/log/nginx/access_json.log json;

      sendfile        on;
      #tcp_nopush     on;

      #keepalive_timeout  0;
      keepalive_timeout  65;

      include /etc/nginx/vhost/*.conf;

      #gzip  on;
}

Logstash配置文件nginx日志采集配置

cp /etc/logstash/logstash-sample.conf  /etc/logstash/conf.d/nginx-log.conf
vi nginx-log.conf      
#写入以下内容
input {
    file{
        path => "/var/log/nginx/access_json.log"  #需要采集的json格式nginx access日志文件路径
        codec => json                             #数据格式
        #start_position => "beginning"            #默认为采集新增内容，begining为从文件开头开始采集
        #type => "nginx_log"                      #类型
    }
}
output {
    #es配置
    elasticsearch {
        hosts => "eshost:9200"      #es地址
        index => "logstash-nginx"   #写入的索引名
        user => elastic             #es用户
        password => espassword      #es密码
    }
    #logstash控制台输入采集到的数据，用于调试
    #stdout {
    #    codec => rubydebug
    #}
}

后台启动logstash服务

#logstash服务路径：/usr/share/logstash/bin/logstash
#后台启动logstash服务，并将内容输出到/var/log/logstash/output_nginx.log文件下
nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log.conf >> /var/log/logstash/output_nginx.log 2>&1 &
#查看logstash服务
ps -ef | grep logstash
#关闭logstash服务
kill -9 logstash进程号

kibana查看配置

选择logstash-nginx索引，创建索引模式
discover面板查看

image.png

Docker安装Logstash采集Nginx日志

docker安装logstash，logstash服务运行在容器内，采用file类型只能采集容器内存在的文件内容，无法采集宿主机文件，需要使用syslog方式采集宿主机日志

nginx配置需要开启syslog推送

access_log syslog:server=接收主机ip:514,facility=local7,tag=nginx,severity=info,nohostname main_json;

最后编辑于：2020.10.23 15:39:27