krkr
10/9/2014 - 7:25 PM

Logstash configuration to parse Scala/Java logs with a codec multine to join exception and stacktrace messages into a single event and a gro

Logstash configuration to parse Scala/Java logs with a codec multine to join exception and stacktrace messages into a single event and a grok filter to extract the fields (timestamp, level, thread, class and message). The log file is parsed from the beginning, rm ~/.sincedb_ * to reindex it.

logstash.conf
content_copyfile_download

# 2014-10-09 10:47:47,358 INFO (pool-4-thread-2) [io.krkr.Pof] Bla bla ...

input {
  file {
    type => "xyz"
    path => "/var/log/xyz/server-xyz.log"
    start_position => "beginning"
    codec => multiline {
      pattern => "^%{TIMESTAMP_ISO8601} "
      negate => true
      what => "previous"
    }
  }
}

filter {
  grok {
    match => [ "message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:level} \(%{NOTSPACE:thread}\) \[%{NOTSPACE:class}\] (?<message>.*)" ]
    overwrite => [ "message" ]
  }
}


output {
  elasticsearch {
        protocol => "http"
        host => "localhost"
        flush_size => 50
  }
}