首页 新闻 赞助 找找看

logstah 从mysql想ES导入数据时,多出一条%{id}的数据

-1
悬赏园豆:5 [已关闭问题] 关闭于 2021-06-22 10:36

用logstash从mysql想ES导入数据时,mysql里面是36条数据,导入成功后,ES显示37条,逐条观察后发现多了一条数据,数据的_id为config文件里面配置的document_id,也就是%{id},id为mysql表的主键。


以下是mysql表结构的部分截图


以下是es导数据后多出来的数据


以下是config配置

input {
    stdin {
    }
    jdbc {
      jdbc_connection_string => "jdbc:mysql://localhost:3306/tpi-servicesystem?characterEncoding=utf8&serverTimezone=GMT%2B8"
      jdbc_user => "root"
      jdbc_password => "123456"
      jdbc_driver_library => "../mysql/mysql-connector-java-8.0.21.jar"
      jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
      jdbc_paging_enabled => "true"
      jdbc_page_size => "50000"
      statement_filepath => "../mysql/cjfqbaseinfo_metadata.sql"
      schedule => "* * * * *"
      type => "cjfqbaseinfo_metadata"
	  lowercase_column_names => false
	  clean_run => false
    }
}


filter {
    json {
        source => "message"
        remove_field => ["message","host","path","message","@timestamp","@version"]
    }
	
}


output {
	elasticsearch {
		hosts => ["localhost:9200"]
		index => "cjfqbaseinfo_metadata"
		document_type => "cjfqbaseinfo_metadata"
		document_id => "%{PYKM}"
		template => "../mysql/cjfqbaseinfo_metadata.json"
		template_name => "cjfqbaseinfo_metadata"
		template_overwrite => true
		doc_as_upsert => true
	}
    stdout {
        codec => json_lines
    }
}

以下是mysql语句

select  *   from cjfqbaseinfo_metadata 

*****以下是ES模板的json

{
  "template": "cjfqbaseinfo_metadata",
  "settings": {
    "index": {
	  "number_of_shards" : 2,
	  "number_of_replicas" : 0,
      "analysis": {
        "analyzer": {
          "default" : {
            "tokenizer" : "ik_max_word"
          },
          "pinyin_analyzer": {
            "type":"custom",
            "tokenizer": "ik_max_word",
            "filter":"my_pinyin"
          },
		  "pinyin_prefix_analyzer": {
			"tokenizer" : "keyword",
			"filter":"my_pinyin_prefix",
			"char_filter":["html_strip"]
		  },
		  "pinyin_suggest_analyzer": {
			"tokenizer" : "keyword",
			"filter":"my_pinyin_suggest",
			"char_filter":["html_strip"]
        },
          "comma_analyzer": {
            "tokenizer": "comma"
          }
        },
        "filter": {
          "my_pinyin": {
            "type": "pinyin",
            "keep_first_letter": true,
            "keep_full_pinyin": true,
            "keep_joined_full_pinyin":true,
            "keep_separate_first_letter": false,
            "limit_first_letter_length": 16,
            "keep_original": true,
            "lowercase": false,
            "trim_whitespace": true,
            "remove_duplicated_term": true
          },
          "my_pinyin_prefix": {
            "type": "pinyin",
			"keep_full_pinyin": false,
			"keep_joined_full_pinyin":true,
			"keep_original": true,
			"lowercase": false,
			"trim_whitespace": true,
			"remove_duplicated_term": true
          },
          "my_pinyin_suggest": {
            "type": "pinyin",
			"keep_full_pinyin": false,
			"keep_joined_full_pinyin":true,
			"keep_original": true,
			"lowercase": false,
			"trim_whitespace": true,
			"remove_duplicated_term": true
        }
        },
        "tokenizer":{
          "comma": {
            "type": "pattern",
            "pattern": ",|,|;|;|、"
          }
        }
      }
    }
  },
  "mappings": {
    "cjfqbaseinfo_metadata": {
      "properties": {
        "IDNO": {
          "type": "integer"
        },
        "PYKM": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            },
            "prefix": {
              "type": "text",
              "analyzer": "pinyin_prefix_analyzer",
              "search_analyzer": "keyword"
            },
            "suggest": {
              "type": "completion",
              "analyzer": "pinyin_suggest_analyzer",
              "search_analyzer": "keyword",
              "preserve_separators": true,
              "preserve_position_increments": true,
              "max_input_length": 50
            }
          },
          "analyzer": "pinyin_analyzer",
          "search_analyzer": "ik_max_word"
        },
	"刊期1": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            },
            "prefix": {
              "type": "text",
              "analyzer": "pinyin_prefix_analyzer",
              "search_analyzer": "keyword"
            },
            "suggest": {
              "type": "completion",
              "analyzer": "pinyin_suggest_analyzer",
              "search_analyzer": "keyword",
              "preserve_separators": true,
              "preserve_position_increments": true,
              "max_input_length": 50
            }
          },
          "analyzer": "pinyin_analyzer",
          "search_analyzer": "ik_max_word"
        },
	"KQ": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            },
            "prefix": {
              "type": "text",
              "analyzer": "pinyin_prefix_analyzer",
              "search_analyzer": "keyword"
            },
            "suggest": {
              "type": "completion",
              "analyzer": "pinyin_suggest_analyzer",
              "search_analyzer": "keyword",
              "preserve_separators": true,
              "preserve_position_increments": true,
              "max_input_length": 50
            },
			 "comma": {
              "type": "text",
              "analyzer": "comma_analyzer",
              "fielddata": true
            }
          },
          "analyzer": "pinyin_analyzer",
          "search_analyzer": "ik_max_word"
        },
        "CBD_403": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            },
            "prefix": {
              "type": "text",
              "analyzer": "pinyin_prefix_analyzer",
              "search_analyzer": "keyword"
            },
            "suggest": {
              "type": "completion",
              "analyzer": "pinyin_suggest_analyzer",
              "search_analyzer": "keyword",
              "preserve_separators": true,
              "preserve_position_increments": true,
              "max_input_length": 50
            }
          },
          "analyzer": "pinyin_analyzer",
          "search_analyzer": "ik_max_word"
        },
        "是否过刊": {
          "type": "integer"
        },
        "sort_id": {
          "type": "integer"
        },"ImportDate": {
          "type": "date"
        }, 
        "unit": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            },
            "prefix": {
              "type": "text",
              "analyzer": "pinyin_prefix_analyzer",
              "search_analyzer": "keyword"
            },
            "suggest": {
              "type": "completion",
              "analyzer": "pinyin_suggest_analyzer",
              "search_analyzer": "keyword",
              "preserve_separators": true,
              "preserve_position_increments": true,
              "max_input_length": 50
            }
          },
          "analyzer": "pinyin_analyzer",
          "search_analyzer": "ik_max_word"
        }
      }
    }
  }
}

代码羡的主页 代码羡 | 初学一级 | 园豆:69
提问于:2021-04-23 10:35
< >
分享
所有回答(2)
0

你好,请问这种情况找到原因了吗?我这边也遇到了。

风吹16沙落1 | 园豆:204 (菜鸟二级) | 2023-05-19 15:45
0

怎么解决?

明天喝可乐 | 园豆:202 (菜鸟二级) | 2023-09-14 18:02
清除回答草稿
   您需要登录以后才能回答,未注册用户请先注册