Elk安装
1.下载安装包
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.2.zip
2.解压安装包
unzip elasticsearch-6.2.2.zip
3.启动ELK
./bin/elasticsearch
3.1权限不足,Root无法启动
Caused by: java.lang.RuntimeException: can not run elasticsearch as root
at org.elasticsearch.bootstrap.Bootstrap.initializeNatives(Bootstrap.java:105) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:172) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:323) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:121) ~[elasticsearch-6.2.2.jar:6.2.2]
解决办法
adduser elk
passwd elk
groupadd elk
usermod -G elk elk
chmod -R 755 elasticsearch-6.2.2
3.2开放所有地址访问
vi config/elasticsearch.yml 开放设置，并且将地址设置为允许所有访问
network.host: 0.0.0.0
3.3虚拟内存设置不足，无法启动
ERROR: [1] bootstrap checks failed
[1]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
解决办法
sysctl -w vm.max_map_count=262144
4.启动成功
访问 http://ip:9200 ,得到正确返回
{
name: "BaYiuYx",
cluster_name: "elasticsearch",
cluster_uuid: "Yg8UQIx1QgazeGmOy6B7lw",
version:
{
number: "6.2.2",
build_hash: "10b1edd",
build_date: "2018-02-16T19:01:30.685723Z",
build_snapshot: false,
lucene_version: "7.2.1",
minimum_wire_compatibility_version: "5.6.0",
minimum_index_compatibility_version: "5.0.0"
},
tagline: "You Know, for Search"
}
安装analysis-ik插件
1.下载并安装
./elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v6.2.2/elasticsearch-analysis-ik-6.2.2.zip
2.重启ELK
[2018-03-20T17:57:22,635][INFO ][o.e.p.PluginsService ] [BaYiuYx] loaded module [tribe]
[2018-03-20T17:57:22,636][INFO ][o.e.p、.PluginsService ] [BaYiuYx] loaded plugin [analysis-ik]
使用Elk+IK搜索
创建索引
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts' -d '
{
"mappings": {
"person": {
"properties": {
"user": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_max_word"
},
"title": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_max_word"
},
"desc": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_max_word"
}
}
}
}
}'
插入数据
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts/person/1' -d '
{
"user": "张三1",
"title": "工程师1",
"desc": "数据库管理1"
}'
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts/person/2' -d '
{
"user": "李四2",
"title": "律师2",
"desc": "公司法务2"
}'
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts/person/3' -d '
{
"user": "王五3",
"title": "经理3",
"desc": "土木工程3"
}'
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts/person/4' -d '
{
"user": "赵六4",
"title": "HR4",
"desc": "人力资源4"
}'
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts/person/5' -d '
{
"user": "胡七5",
"title": "采购5",
"desc": "供应链5"
}'
curl -H "Content-Type: application/json" -X PUT 'localhost:9200/accounts/person/6' -d '
{
"user": "习六6",
"title": "工程师6",
"desc": "库管 社区6"
}'
查询数据
curl -H "Content-Type: application/json" 'localhost:9200/accounts/person/_search' -d '
{
"query" : { "match" : { "desc" : "社6" }}
}'
测试分词
原始分词测试
curl -H "Content-Type: application/json" 'http://localhost:9200/_analyze?pretty=true' -d '{"text":"这里是好记性不如烂笔头感叹号的博客园"}'
IK插件分词测试
curl -H "Content-Type: application/json" 'http://localhost:9200/_analyze?pretty=true' -d '{"text":"这里是好记性不如烂笔头感叹号的博客园","analyzer": "ik_smart"}'
[root@dawner config]# curl -H "Content-Type: application/json" 'http://localhost:9200/_analyze?pretty=true' -d '{"text":"这里是好记性不如烂笔头感叹号的博客园","analyzer": "ik_smart"}'
{
"tokens" : [
{
"token" : "这里是",
"start_offset" : 0,
"end_offset" : 3,
"type" : "CN_WORD",
"position" : 0
},
{
"token" : "好",
"start_offset" : 3,
"end_offset" : 4,
"type" : "CN_CHAR",
"position" : 1
},
安装拼音pinyin插件
安装
./elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-pinyin/releases/download/v6.2.2/elasticsearch-analysis-pinyin-6.2.2.zip
测试拼音分词
curl -H "Content-Type: application/json" 'http://localhost:9200/_analyze?pretty=true' -d '{"text":"张学友","analyzer": "pinyin"}'
{
"tokens" : [
{
"token" : "zhang",
"start_offset" : 0,
"end_offset" : 0,
"type" : "word",
"position" : 0
},
{
"token" : "zxy",
"start_offset" : 0,
"end_offset" : 0,
"type" : "word",
"position" : 0
安装logstash
下载
wget https://artifacts.elastic.co/downloads/logstash/logstash-6.2.2.zip
解压缩
unzip logstash/logstash-6.2.2.zip
下载mysql jdbc连接jar
增加logstatsh配置文件
jdbc.conf
input {
stdin {
}
jdbc {
# mysql jdbc connection string to our backup databse
jdbc_connection_string => "jdbc:mysql://xxxxx:3306/demo"
# the user we wish to excute our statement as
jdbc_user => "xxxxx"
jdbc_password => "xxxxx"
# the path to our downloaded jdbc driver
jdbc_driver_library => "/hdd2/backup/software/elk/logstash/logstash-6.2.2/external/lib/mysql-connector-java-5.1.21.jar"
# the name of the driver class for mysql
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement_filepath => "../external/jdbc.sql"
schedule => "* * * * *"
type => "jdbc"
}
}
filter {
json {
source => "message"
remove_field => ["message"]
}
}
output {
elasticsearch {
hosts => "118.89.16.61:9200"
index => "demo"
document_type => "documents"
document_id => "%{id}"
}
stdout {
codec => json_lines
}
}
jdbc.sql
select id,title,content,author,create_time,description from document t
启动logstatsh
./logstash -f ../external/jdbc.conf
检查Elk索引
curl -H "Content-Type: application/json" 'localhost:9200/demo/documents/_search'
测试分页查询
curl -H "Content-Type: application/json" 'localhost:9200/demo/documents/_search' -d '{ "size": 1, "from": 0, "query" : { "match" : { "content" : "首次" }}, "highlight": { "fields": { "content": { "pre_tags": "",
"post_tags": "" } } }}'
etc
http://blog.csdn.net/q15150676766/article/details/76446033
