mysql数据全量写入elasticsearch
报错脏数据
ESWriter$Job - response code: [200] error :[One or more of the items in the Bulk request failed, check BulkResult.getItems() for more information.]
原因
job脚本中type类型错误
type属性必须显示指定为_doc，不能使用默认的default
{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"column": [
"id",
"content_id",
"name",
"actor_display",
"writer_display",
"original_country",
"language",
"relase_year",
"status",
"kpeople",
"director",
"script_writer",
"type",
"tags",
"start_level",
"awards"
],
"connection": [
{
"jdbcUrl": [
"jdbc:mysql://139.196.36.113:3306/gzdpdb20200907"
],
"table": [
"t_content"
]
}
],
"password": "CSQAPukka",
"username": "root"
}
},
"writer": {
"name": "elasticsearchwriter",
"parameter": {
"endpoint": "http://192.168.6.108:9200",
"accessId": "root",
"accessKey": "admin123",
"index": "t_content",
"type": "_doc",
"cleanup": false,
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 0
}
},
"discovery": false,
"batchSize": 1000,
"splitter": ",",
"column": [
{
"name": "id",
"type": "long"
},
{
"name": "contentId",
"type": "keyword"
},
{
"name": "name",
"type": "keyword"
},
{
"name": "actorDisplay",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "writerDisplay",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "originalCountry",
"type": "keyword"
},
{
"name": "language",
"type": "keyword"
},
{
"name": "relaseYear",
"type": "keyword"
},
{
"name": "status",
"type": "keyword"
},
{
"name": "kpeople",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "director",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "scriptwriter",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "type",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "tags",
"type": "text",
"analyzer": "ik_smart"
},
{
"name": "startLevel",
"type": "keyword"
},
{
"name": "awards",
"type": "text",
"analyzer": "ik_smart"
}
]
}
}
}
]
}
}
一次性能记录
任务启动时间 2020-12-07 13：55：06
任务结束时间 2020-12-07 14：18：47
任务总计耗时 1421s
任务平均流量 78.81KB/s
记录写入速度 1313rec/s
读出记录总数 1865344
读写失败总数 0
