ELK架构图例
紧接上文：https://www.jianshu.com/p/af308f558c7d
logstash安装
购买云主机
| 主机 | IP地址 | 配置 |
|---|---|---|
| logstash | 192.168.1.47 | 最低配置2核2G |
logstash云主机安装
[root@logstash ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
192.168.1.47 logstash
[root@logstash ~]# yum install -y java-1.8.0-openjdk logstash
基础配置样例
[root@logstash ~]# ln -s /etc/logstash /usr/share/logstash/config
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin {}
}
filter{ }
output{
stdout{}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
插件与调试格式
使用json格式字符串测试 {"a":"1", "b":"2", "c":"3"}
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin { codec => "json" }
}
filter{ }
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
官方手册地址
https://www.elastic.co/guide/en/logstash/current/index.html
input file插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb"
}
}
filter{ }
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# rm -rf /var/lib/logstash/plugins/inputs/file/.sincedb_*
[root@logstash ~]# /usr/share/logstash/bin/logstash
filter grok插件
正则表达式分组匹配格式: (?<名字>正则表达式)
正则表达式宏调用格式: %{宏名称:名字}
宏文件路径
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns
[root@logstash ~]# echo '192.168.1.252 - - [29/Jul/2020:14:06:57 +0800] "GET /info.html HTTP/1.1" 200 119 "-" "curl/7.29.0"' >/tmp/c.log
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}
}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
output elasticsearch插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}
}
output{
stdout{ codec => "rubydebug" }
elasticsearch {
hosts => ["es-0004:9200", "es-0005:9200"]
index => "weblog-%{+YYYY.MM.dd}"
}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
浏览器打开 head 插件，通过 web 页面浏览验证
http://公网IP:9200/_plugin/head/
web 页面浏览验证
filebeat配置
logstash beats插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin { codec => "json" }
file{
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb"
}
beats {
port => 5044
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}
}
output{
stdout{ codec => "rubydebug" }
elasticsearch {
hosts => ["es-0004:9200", "es-0005:9200"]
index => "weblog-%{+YYYY.MM.dd}"
}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
web服务安装filebeat
[root@web ~]# yum install -y filebeat
[root@web ~]# vim /etc/filebeat/filebeat.yml
24: enabled: true
28: - /var/log/httpd/access_log
45: fields:
46: my_type: apache
148, 150 注释掉
161: output.logstash:
163: hosts: ["192.168.1.47:5044"]
180, 181, 182 注释掉
[root@web ~]# grep -Pv "^\s*(#|$)" /etc/filebeat/filebeat.yml
[root@web ~]# systemctl enable --now filebeat
