logstah 从mysql想ES导入数据时，多出一条%{id}的数据
-1
悬赏园豆：5
[已关闭问题]
关闭于 2021-06-22 10:36
用logstash从mysql想ES导入数据时，mysql里面是36条数据，导入成功后，ES显示37条，逐条观察后发现多了一条数据，数据的_id为config文件里面配置的document_id,也就是%{id}，id为mysql表的主键。
以下是mysql表结构的部分截图
以下是es导数据后多出来的数据
以下是config配置
input {
stdin {
}
jdbc {
jdbc_connection_string => "jdbc:mysql://localhost:3306/tpi-servicesystem?characterEncoding=utf8&serverTimezone=GMT%2B8"
jdbc_user => "root"
jdbc_password => "123456"
jdbc_driver_library => "../mysql/mysql-connector-java-8.0.21.jar"
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement_filepath => "../mysql/cjfqbaseinfo_metadata.sql"
schedule => "* * * * *"
type => "cjfqbaseinfo_metadata"
lowercase_column_names => false
clean_run => false
}
}
filter {
json {
source => "message"
remove_field => ["message","host","path","message","@timestamp","@version"]
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "cjfqbaseinfo_metadata"
document_type => "cjfqbaseinfo_metadata"
document_id => "%{PYKM}"
template => "../mysql/cjfqbaseinfo_metadata.json"
template_name => "cjfqbaseinfo_metadata"
template_overwrite => true
doc_as_upsert => true
}
stdout {
codec => json_lines
}
}
以下是mysql语句
select * from cjfqbaseinfo_metadata
*****以下是ES模板的json
{
"template": "cjfqbaseinfo_metadata",
"settings": {
"index": {
"number_of_shards" : 2,
"number_of_replicas" : 0,
"analysis": {
"analyzer": {
"default" : {
"tokenizer" : "ik_max_word"
},
"pinyin_analyzer": {
"type":"custom",
"tokenizer": "ik_max_word",
"filter":"my_pinyin"
},
"pinyin_prefix_analyzer": {
"tokenizer" : "keyword",
"filter":"my_pinyin_prefix",
"char_filter":["html_strip"]
},
"pinyin_suggest_analyzer": {
"tokenizer" : "keyword",
"filter":"my_pinyin_suggest",
"char_filter":["html_strip"]
},
"comma_analyzer": {
"tokenizer": "comma"
}
},
"filter": {
"my_pinyin": {
"type": "pinyin",
"keep_first_letter": true,
"keep_full_pinyin": true,
"keep_joined_full_pinyin":true,
"keep_separate_first_letter": false,
"limit_first_letter_length": 16,
"keep_original": true,
"lowercase": false,
"trim_whitespace": true,
"remove_duplicated_term": true
},
"my_pinyin_prefix": {
"type": "pinyin",
"keep_full_pinyin": false,
"keep_joined_full_pinyin":true,
"keep_original": true,
"lowercase": false,
"trim_whitespace": true,
"remove_duplicated_term": true
},
"my_pinyin_suggest": {
"type": "pinyin",
"keep_full_pinyin": false,
"keep_joined_full_pinyin":true,
"keep_original": true,
"lowercase": false,
"trim_whitespace": true,
"remove_duplicated_term": true
}
},
"tokenizer":{
"comma": {
"type": "pattern",
"pattern": ",|，|;|；|、"
}
}
}
}
},
"mappings": {
"cjfqbaseinfo_metadata": {
"properties": {
"IDNO": {
"type": "integer"
},
"PYKM": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"prefix": {
"type": "text",
"analyzer": "pinyin_prefix_analyzer",
"search_analyzer": "keyword"
},
"suggest": {
"type": "completion",
"analyzer": "pinyin_suggest_analyzer",
"search_analyzer": "keyword",
"preserve_separators": true,
"preserve_position_increments": true,
"max_input_length": 50
}
},
"analyzer": "pinyin_analyzer",
"search_analyzer": "ik_max_word"
},
"刊期1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"prefix": {
"type": "text",
"analyzer": "pinyin_prefix_analyzer",
"search_analyzer": "keyword"
},
"suggest": {
"type": "completion",
"analyzer": "pinyin_suggest_analyzer",
"search_analyzer": "keyword",
"preserve_separators": true,
"preserve_position_increments": true,
"max_input_length": 50
}
},
"analyzer": "pinyin_analyzer",
"search_analyzer": "ik_max_word"
},
"KQ": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"prefix": {
"type": "text",
"analyzer": "pinyin_prefix_analyzer",
"search_analyzer": "keyword"
},
"suggest": {
"type": "completion",
"analyzer": "pinyin_suggest_analyzer",
"search_analyzer": "keyword",
"preserve_separators": true,
"preserve_position_increments": true,
"max_input_length": 50
},
"comma": {
"type": "text",
"analyzer": "comma_analyzer",
"fielddata": true
}
},
"analyzer": "pinyin_analyzer",
"search_analyzer": "ik_max_word"
},
"CBD_403": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"prefix": {
"type": "text",
"analyzer": "pinyin_prefix_analyzer",
"search_analyzer": "keyword"
},
"suggest": {
"type": "completion",
"analyzer": "pinyin_suggest_analyzer",
"search_analyzer": "keyword",
"preserve_separators": true,
"preserve_position_increments": true,
"max_input_length": 50
}
},
"analyzer": "pinyin_analyzer",
"search_analyzer": "ik_max_word"
},
"是否过刊": {
"type": "integer"
},
"sort_id": {
"type": "integer"
},"ImportDate": {
"type": "date"
},
"unit": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"prefix": {
"type": "text",
"analyzer": "pinyin_prefix_analyzer",
"search_analyzer": "keyword"
},
"suggest": {
"type": "completion",
"analyzer": "pinyin_suggest_analyzer",
"search_analyzer": "keyword",
"preserve_separators": true,
"preserve_position_increments": true,
"max_input_length": 50
}
},
"analyzer": "pinyin_analyzer",
"search_analyzer": "ik_max_word"
}
}
}
}
}
所有回答(2)
0
你好，请问这种情况找到原因了吗？我这边也遇到了。
0
怎么解决？
