Logstash/Graylog不会在数据节点关闭的情况下写入ES

Question

我们遇到了一个问题，我们的一个Elasticsearch数据节点在AWS中完全死亡。即使有了第二个节点，我们也很难将集群恢复联机。它最初是以yellow状态出现的，我们可以检索数据，但不能写入数据。

弹性搜索误差

2016-09-12 13:16:42,312discovery.zen.ping.unicast未能将ping发送给{#Cloudy-I-cb397257-0}{1.1.1.1}{1.1.1.1:9300}13:16:42,312discovery.zen.ping.unicast嵌套: IllegalStateExceptionreceived ping请求，但未启动；原因是: java.lang.IllegalStateException:未启动时接收到的ping请求

测井误差

{:timestamp=>"2016-09-12T15:48:15.898000-0400"，:message=>#900592725，:events_consumed=>900732955，:worker_count=>4，:inflight_count=>0，：worker_states=>{：status=>“睡眠”，:alive=>true，:index=>0，:inflight_count=>0}，{：status=>“睡眠”，:alive=>true，:index=>1，:inflight_count=>0}，{：status=>“睡眠”，:alive=>true，:index=>2，：status=>}，{en20 19#，en20 20#，:inflight_count=>0}、:output_info=>[{:type=>"gelf“、：config=>{”主机“”=>“127.0.0.1”、：config=> :is_multi_worker=>false、:events_received=>900592725、:workers=>"127.0.0.1“、codec=>"UTF-8">、workers=>1、port=>12201、chunksize=>1420、sender=>"%{host}”、level=>“%{：config=>}”、"INFO“、ship_metadata=>true、ship_tags=>true、level=>”@timestamp“，"@version“、”严重性“、"host”、"source_host“、"source_path”、"short_message“、full_message=>"%{message}”、short_message=>“short_message”>>、：busy_workers=>0}、:thread_info=>[{"thread_id"=>17、“name”“=>”mainnil、"backtrace"=>[".../vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'", "[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in接受‘，".../vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:112:in run_server'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:84:inrun’".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:342:in inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:336:instart_input‘]，"blocked_on"=>nil，状态“=>”睡眠，"current_call"=>".../vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步’，".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:inworker_loop‘，".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步’}，{"thread_id"=>19，“名称”“=>”main>worker1，“plugin”=>“LogStash：：Filters:Grok”，{"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/"，“匹配”=>{“消息”“=>”%{SYSLOGBASE2}}，"backtrace"=>[".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'"，".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:instart_workers‘]，"blocked_on"=>nil，状态“=>”睡眠，"current_call"=>".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步’，".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:inworker_loop‘，".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步’}，{"thread_id"=>21，“名称”“=>”main>worker3，“plugin”=>“LogStash：：Filters:Grok”，{"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/"，“匹配”=>{“消息”“=>”%{SYSLOGBASE2}}，"backtrace"=>[".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'"，".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:instart_workers‘]，"blocked_on"=>nil，“状态”“=>”睡眠，"current_call"=>".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}], :stalling_threads_info=>[{"thread_id"=>17, "name"=>"[main]<tcp", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:inaccept’}，{"thread_id"=>18，“名称”“=>”main>worker0，"plugin"=>"LogStash::Filters::Grok"，“=>{”消息“=>”%{SYSLOGBASE2}}，"current_call"=>".../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步‘}，{"thread_id"=>20，“名称”“=>”main>worker2，"plugin"=>"LogStash::Filters::Grok"，{"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/"，“match”=>{“消息”“=>”%{SYSLOGBASE2}}，synchronize'"}, {"thread_id"=>21, "name"=>"[main]>worker3", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步‘}}>，:level=>:warn}

Answer 1

问题是我们在ES集群中使用了AWS发现。即使我们从我们使用的安全组中删除了死实例(在AWS中仍然可见，只是无法访问)，我们也没有重新启动所有剩余的ES节点。一旦我们这样做了，一切又开始运作了。