ELK中的Logstash优化

优化前

Logstash filter 中的grok日志拆分使用的是正则表达式，CPU利用率太高

# For detail structure of this file  
# Set: https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html  
input {  
  # For detail config for log4j as input,   
  # See: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-log4j.html  
  file {  
    path => "/data/log/beat/*.log"
    type => "beat"
  }  
}

filter {  
  #Only matched data are send to output. 
    grok {  
      json {
         source => "message"
         target => "message"
      }
      mutate {
        add_field  => {
           "hostname" => "%{[message][beat][hostname]}"
        }
     }
     ruby {
       code => "event.set('message',event.get('message')['message'])"
     }
     grok {       
      match => {"%{TIMESTAMP_ISO8601:timestamp}\s+%{WORD:level}\s+\[%{NOTSPACE:service},%{DATA:trace},%{DATA:span},%{DATA:exportable}\]\s+%{INT:pid}\s+---\s+\[%{GREEDYDATA:thread}\]\s+%{NOTSPACE:class}\s+:\s+%{GREEDYDATA:message}"} 
     } 
}  

output {  
  # For detail config for elasticsearch as output,   
  # See: https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html  
 
	  elasticsearch {  
	    action => "index"          #The operation on ES  
	    hosts  => "elasticsearch1:9200"   #ElasticSearch host, can be array.  
	    index  => "applog-%{+YYYY.MM.dd}"         #The index to write data to.  
	    user => elastic  
	    password => changeme  
	  }  
}  

优化后

使用ruby代替grok，cpu效果非常明显

# For detail structure of this file  
# Set: https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html  
input {  
  # For detail config for log4j as input,   
  # See: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-log4j.html  
  file {  
    path => "/data/log/beat/*.log"
    type => "beat"
  }  
}

filter {  
  #Only matched data are send to output. 
  
      json {
         source => "message"
         target => "message"
      }
      mutate {
        add_field  => {
           "hostname" => "%{[message][beat][hostname]}"
        }
     }
     ruby {
       code => "
	msg = event.get('message')['message']
	idx = msg.index(' ')
	idx = msg.index(' ',idx+1)
	timestamp = msg[0,idx]
	event.set('timestamp',timestamp)
	msg =  msg[idx..-1]
	msg = msg.lstrip()
	idx = msg.index(' ')
	level = msg[0,idx]
	event.set('level',level)

	msg =  msg[idx..-1]
	msg = msg.lstrip()
	idx = msg.index(' ')
	traces =  msg[0,idx]
	traces = traces.delete('[]')

	msg =  msg[idx..-1]
	msg = msg.lstrip()

	idx = traces.index(',')
	service =  traces[0,idx]
	event.set('service',service)

	traces = traces[idx+1..-1]
	idx = traces.index(',')
	trace =  traces[0,idx]
	event.set('trace',trace)

	traces =  traces[idx+1..-1]
	idx = traces.index(',')
	span =  traces[0,idx]
	event.set('span',span)


	lidx = msg.index('[')
	ridx = msg.index(']')
	thread = msg[lidx+1...ridx]
	event.set('thread',thread)

	msg = msg[ridx+1..-1]
	msg = msg.lstrip()

	idx = msg.index(' ')
	cls =  msg[0,idx]
	event.set('class',cls)

	msg = msg[idx+1..-1]

	idx = msg.index(':')
	msg = msg[idx+1..-1]
	msg = msg.lstrip()
	event.set('message',msg)
       "
}  

output {  
  # For detail config for elasticsearch as output,   
  # See: https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html    
	  elasticsearch {  
	    action => "index"          #The operation on ES  
	    hosts  => "elasticsearch1:9200"   #ElasticSearch host, can be array.  
	    index  => "applog-%{+YYYY.MM.dd}"         #The index to write data to.  
	    user => elastic  
	    password => changeme  
	  }  
}