作者：李建生
归档：部署文档

第1章 安装准备

由于Elasticsearch、Logstash、Kibana均不能以root账号运行。
但是Linux对非root账号可并发操作的文件、线程都有限制。
所以，部署ELK相关的机器都要调整：

1.1 安装jdk

yum install java-1.8.0-openjdk  java-1.8.0-openjdk-devel  -y
[root@elk3 tools]# echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.51.x86_64 \
> export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar\
> export PATH=$PATH:$JAVA_HOME/bin" >> /etc/profile 
[root@elk2 ~]# source  /etc/profile
[root@elk3 tools]# java -version
openjdk version "1.8.0_161"
OpenJDK Runtime Environment (build 1.8.0_161-b14)
OpenJDK 64-Bit Server VM (build 25.161-b14, mixed mode)

1.2 修改文件限制

# 修改系统文件
vim /etc/security/limits.conf
#增加的内容
* soft nofile 65536
* hard nofile 65536
* soft nproc 2048
* hard nproc 4096

1.3 调整进程数

#修改系统文件
vim /etc/security/limits.d/20-nproc.conf
#调整成以下配置
*          soft    nproc     4096
root       soft    nproc     unlimited

1.4 调整虚拟内存&最大并发连接

#修改系统文件
vim /etc/sysctl.conf
#增加的内容
vm.max_map_count=655360
fs.file-max=655360
vm.max_map_count=262144

1.5 创建普通用户

groupadd elsearch  
useradd elsearch -g elsearch -p 123456
chown -R elsearch.elsearch /server/tools/elasticsearch-6.2.3

1.6 创建日志和数据目录

mkdir -p /var/data/elasticsearch &&  chown -R elsearch.elsearch /var/data/elasticsearch
mkdir -p /var/log/elasticsearch &&   chown -R elsearch.elsearch /var/log/elasticsearch

第2章 部署ELK

Elasticsearch是一个高度可扩展的开源全文搜索和分析引擎。它允许您快速，近实时地存储，搜索和分析大量数据。它通常用作支持具有复杂搜索功能和需求的应用程序的底层引擎/技术。

2.1 安装elasticsearch

2.1.1 配置elasticsearch配置文件

cat >/server/tools/elasticsearch-6.2.3/config/elasticsearch.yml <<END
cluster.name: ES
node.name: ES0
path.data: /var/data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.19
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["10.0.0.19:9300", "10.0.0.21:9300","10.0.0.22:9300"]
discovery.zen.minimum_master_nodes: 2
END
cat >/server/tools/elasticsearch-6.2.3/config/elasticsearch.yml <<END
cluster.name: ES
node.name: ES1
path.data: /var/data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.21
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["10.0.0.19:9300", "10.0.0.21:9300","10.0.0.22:9300"]
discovery.zen.minimum_master_nodes: 2
END

cat >/server/tools/elasticsearch-6.2.3/config/elasticsearch.yml <<END
cluster.name: ES
node.name: ES2
path.data: /var/data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.22
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["10.0.0.19:9300", "10.0.0.21:9300","10.0.0.22:9300"]
discovery.zen.minimum_master_nodes: 2
END

2.1.2 启动

elsearch@elk1 elasticsearch-6.2.3]$ ./bin/elasticsearch -d #后台启动

2.2 安装配置Logstash

2.2.1 安装准备

[root@elk3 ~]# mkdir -p /server/tools
[root@elk3 ~]# cd /server/tools/
[root@elk3 tools]# tar xf  kibana-6.2.3-linux-x86_64.tar.gz 
[root@elk3 tools]# tar xf  logstash-6.2.3.tar.gz
groupadd elsearch  
useradd elsearch -g elsearch -p 123456
chown -R elsearch.elsearch /server/tools/

2.2.2 配置Logstash

数据&日志目录
#创建Logstash数据目录
#创建Logstash日志目录
mkdir -p /var/data/logstash &&  chown -R elsearch.elsearch /var/data/logstash
mkdir -p /var/logs/logstash &&   chown -R elsearch.elsearch /var/logs/logstash
配置数据&日志目录
#打开目录
cd /server/tools/logstash-6.2.3/
#修改配置
vim config/logstash.yml
#增加以下内容
path.data: /var/data/logstash
path.logs: /var/logs/logstash
配置Redis&Elasticsearch
[root@elk3 config]# cat system.conf 
input {
    file {
        path => "/var/log/*"
        type => "system"
        start_position => "beginning"
    }
}
output {
    elasticsearch {
        hosts => ["10.0.0.21:9200","10.0.0.19:9200","10.0.0.22:9200"]
        index => "system-%{+YYYY.MM.dd}"
    }
}
[root@elk3 config]#

注：该配置就是从redis中读取数据，然后写入指定的elasticsearch

2.2.3 启动logstash

切换普通用户
[root@elk3 config]# su - elsearch
#进入Logstash根目录
[elsearch@elk3 ~]$ cd /server/tools/logstash-6.2.3/
#启动
/bin/logstash -f config/system.conf
启动成功后，在启动输出的最后一行会看到如下信息：
[2018-04-03T10:54:35,819][INFO ][logstash.pipeline        ] Pipeline started succesfully {:pipeline_id=>"main", :thread=>"#<Thread:0x15a6d6d9 run>"}
[2018-04-03T10:54:36,086][INFO ][logstash.agent           ] Pipelines running {:count=>1, :pipelines=>["main"]}

2.3 Kibana 配置

2.3.1 安装准备

[root@elk3 tools]# tar xf kibana-6.2.3-linux-x86_64
chown elsearch:elsearch /server/tools/kibana-6.2.3-linux-x86_64

2.3.2 修改配置

#进入kibana根目录
cd /server/tools/kibana-6.2.3-linux-x86_64
#修改配置
vi config/kibana.yml
#修改以下内容
[root@elk3 kibana-6.2.3-linux-x86_64]# egrep -v "^#|^$" config/kibana.yml 
server.port: 5601
server.host: "10.0.0.24"
elasticsearch.url: "http://10.0.0.21:9200"
[root@elk3 kibana-6.2.3-linux-x86_64]#

2.3.3 启动

切换普通用户启动
[root@elk3 kibana-6.2.3-linux-x86_64]# su - elsearch 
 [elsearch@elk3 ~]$ cd /server/tools/kibana-6.2.3-linux-x86_64/
[elsearch@elk3 kibana-6.2.3-linux-x86_64]$ bin/kibana

2.3.4 浏览器访问

http://10.0.0.24:5601

第3章 Filebeat配置

系统Filebeat模块收集并分析由基于Unix / Linux的常见发行版的系统日志记录服务创建的日志。

3.1 安装

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.2.3-x86_64.rpm
rpm -vi filebeat-6.2.3-x86_64.rpm
安装插件
bin/elasticsearch-plugin install ingest-geoip
bin/elasticsearch-plugin install ingest-user-agent

3.2 修改/etc/filebeat/filebeat.yml以设置连接信息

output.elasticsearch:
  hosts: ["<es_url>"]
  username: "elastic"
  password: "<password>"
setup.kibana:
  host: "<kibana_url>"

3.3 启用并配置系统模块

filebeat modules enable system
修改/etc/filebeat/modules.d/system.yml文件中的设置。

3.4 启动Filebeat

filebeat setup
systemctl restart filebeat.service

3.5 配置nginx和mysql日志查询

#vim /etc/filebeat/modules.d/mysql.yml
- module: mysql
  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/data/mysql/czypweb.err*"]

  # Slow logs
  slowlog:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
var.paths: ["/data/mysql/mysql-slow.log*"]
#vim /etc/filebeat/modules.d/nginx.yml

- module: nginx
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/usr/local/openresty/nginx/logs/access.log*"]

  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/usr/local/openresty/nginx/logs/error.log*"]
修改filebeat配置文件开启模块功能

第4章 System metrics

系统Metricbeat模块从主机收集CPU，内存，网络和磁盘统计信息。它收集系统范围的统计信息以及每个进程和每个文件系统的统计信息。

4.1 下载

curl -L -O https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-6.2.3-x86_64.rpm
sudo rpm -vi metricbeat-6.2.3-x86_64.rpm

4.2 配置

/etc/metricbeat/metricbeat.yml
metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s
output.elasticsearch:
  hosts: ["<es_url>"]
  username: "elastic"
  password: "<password>"
setup.kibana:
  host: "<kibana_url>"

4.3 启用并配置系统模块

metricbeat modules enable system

4.4 启动Metricbeat

metricbeat setup
service metricbeat start
systemctl enable metricbeat.==service开机自启==