logstash-kafka2elastic

nohup bin/logstash -f config/news_ai_result.conf  > news_ai_result.log &
--------------------------------------------------------------------------------------------------------
[2018-08-15 15:56:54 hadoop@ecsc2901 /apps/logstash-ai/config]$ vi news_ai_result.conf 
input {
        kafka {
                bootstrap_servers => "172.17.129.101:9092,172.17.129.102:9092,172.17.129.103:9092,172.17.129.104:9092,172.17.129.105:9092,172.17.129.106:9092,172.17.129.107:9092,172.17.129.108:9092,172.17.129.109:9092"
                topics => ["news_ai_result"]
                group_id => "logstash_news_ai_result"
                auto_commit_interval_ms => "5000"
                auto_offset_reset => "earliest"
                max_poll_records => "100"
        }
}

filter {
        json {
                source => "message"
                remove_field => "message"
                remove_field => "contenthtml"
        }
        date {
                match => ["extractdate", "yyyy-MM-dd HH:mm:ss"]
        }
}
output {
        #elasticsearch {
        #        hosts => "172.17.129.102:9200"
        #        index => "news_ai_result-%{+YYYY.MM.dd}"
        #        flush_size => 50000
        #}

        elasticsearch {
                hosts => ["172.17.129.101:9200","172.17.129.102:9201","172.17.129.103:9202","172.17.129.104:9203","172.17.129.104:9204"]
                index => "news_ai_result"
                document_id => "%{rowkey}"
                template_overwrite => true
                template => "/apps/mapping/news_ai_result.json"
                template_name=> "index_news_ai_result"
        }
}
-------------------------------------------------------------------------------------------------
[2018-08-15 15:57:34 hadoop@ecsc2901 /apps/logstash-ai/config]$ vi /apps/mapping/news_ai_result.json 

{
        "template" : "index_news_ai_result*",
        "settings" : {
                "index.number_of_shards" : 5,
                "number_of_replicas" : 1,
                "index.refresh_interval" : "1s"
        },
        "dynamic_date_formats":["yyyy-MM-dd HH:mm:ss"],
        "dynamic":false,
        "mappings": {
                "logs": {
                        "properties": {
                                "author":{"type":"string","index":"not_analyzed","store":true,"fielddata":{"format":"fst"}},
                                "cleandate":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "extractdate":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "extracttype":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "publictime":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "source":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "title":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "dataflag": {"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "downloaddate": {"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                "enumber": {"type": "string","index": "not_analyzed", "store": true,"fielddata": {"format": "fst"}},
                                "extractflag": {"type": "string","index": "not_analyzed", "store":true,"fielddata": {"format": "fst"}},
                                "rowkey": {"type": "string","index": "not_analyzed", "store": true, "fielddata": {"format": "fst"}},
                                "url": {"type": "string","index": "not_analyzed", "store": true, "fielddata": {"format": "fst"}},
                                "domain": {"type": "string","index": "not_analyzed", "store": true, "fielddata": {"format": "fst"}}
                        }
                }
        }
}