logstash-7.1.0 安装 & 测试数据导入 ES

wget https://artifacts.elastic.co/downloads/logstash/logstash-7.1.0.tar.gz
tar -zxvf logstash-7.1.0.tar.gz

下载测试数据

wget http://files.grouplens.org/datasets/movielens/ml-20m.zip
unzip ml-20m.zip

编辑.conf文件

cd /opt/logstash-7.1.0/config
vim  logstash.conf
 
input {
  file {
    path => "/opt/data/ml-20m/movies.csv" #编辑为你自己数据存储目录
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}
filter {
  csv {
    separator => ","
    columns => ["id","content","genre"]
  }

  mutate {
    split => { "genre" => "|" }
    remove_field => ["path", "host","@timestamp","message"]
  }

  mutate {
    split => ["content", "("]
    add_field => { "title" => "%{[content][0]}"}
    add_field => { "year" => "%{[content][1]}"}
  }

#  mutate {

#    gsub => [
#
#      "year", "\\)", ""
#    ]
#  }
  mutate {
    convert => {
      "year" => "integer"
    }
    strip => ["title"]
    remove_field => ["path", "host","@timestamp","message","content"]
  }
}

output {
   elasticsearch {
     hosts => "http://es-003:9200"# 编辑为你自己的es集群地址
     index => "movies"
     document_id => "%{id}"
   }
  stdout {}
}

导入数据到ES

#启动Elasticsearch实例，然后启动 logstash，并制定配置文件导入数据
bin/logstash -f  /opt/logstash-7.1.0/config/logstash.conf