elyotechgit
6/7/2016 - 7:14 AM
problems with ES or logstash
problems with ES or logstash
Here is some data.
--- PROD AWS --- at MASTER-Reboot-i-9e05b223-~ : /opt/logstash/bin/logstash -f /etc/logstash/conf.d/ TESTENV
Settings: Default pipeline workers: 1
Pipeline main started
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 1","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 2","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 3","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 4","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 5","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 6","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 7","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 8","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 9","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc 10","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []
all of them have 2 differents session_id
this is my /conf.d/output.conf
output {
if "" in [logsearch][action]
{
stdout { }
elasticsearch {
hosts => ["172.30.1.247","172.30.5.99"]
template_overwrite => true
document_id => "%{[logsearch][sessionId]}"
}
}
else
{
stdout { }
elasticsearch {
hosts => ["172.30.1.247","172.30.5.99"]
template_overwrite => true
}
}
}
this is my search_log.conf
input {
file {
path => "/var/www/travelyo/smartair/app/logs/search.log"
type => "search-log"
sincedb_path => "/opt/logstash/prod-log-sincedb"
sincedb_write_interval => 5
}
}
filter {
if [type] == "search-log" {
grok {
match => {"message" => "\[%{TIMESTAMP_ISO8601:timestamp}\] %{WORD:module}.%{WORD:level}: (?<log_message>.*) \[\] \[\]" }
add_field => [ "received_from", "%{host}" ]
}
json {
source => "log_message"
target => "logsearch"
remove_field=>["log_message"]
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Asia/Jerusalem"
}
}
if ( "" in [logsearch][timeFromNow]) {
mutate {
add_field => {
"[logsearch][timeFromNowInt]" => "%{[logsearch][timeFromNow]}"
"[logsearch][numeric_value]" => "integer"
}
convert => {
"[logsearch][timeFromNowInt]" => "integer"
"[logsearch][timeFromNow]" => "integer"
}
}
}
}
here is how I've added the logs in the logstash
#!/bin/bash
for i in `seq 1 10`
do
first='2010-06-08T06:32:43.000Z MASTER-Reboot-i-9e05b223 [2010-06-08 09:32:43] search.DEBUG: {"sessionId":"abc'
first="$first $i"
first=$first'","logAction":"blabl","timeFromNow":"11","searchFinished":"true","maxDurationReached":"falsefalse","test":"1","site":"smartair.co.il","siteId":67,"client-ip":"192.168.255.6"} [] []'
echo $first >> /var/www/travelyo/smartair/app/logs/search.log
done
and this is the count from logstash
GET /logstash-2010.06.08/_count
{
"count": 20,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
}
}
and doing the _search, I saw the 10 first element from abc 1 to abc 10, and then repeating again. I'm sure there is only one instance of logstash running. as after doing CTRL+C and running again the small scripts I didn't had new docs.
the mapping
"logstash-2010.06.08": {
"mappings": {
"search-log": {
"_all": {
"enabled": true,
"omit_norms": true
},
"dynamic_templates": [
{
"message_field": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": {
"format": "disabled"
},
"type": "string"
},
"match": "message",
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": {
"format": "disabled"
},
"type": "string",
"fields": {
"raw": {
"index": "not_analyzed",
"ignore_above": 256,
"type": "string"
}
}
},
"match": "*",
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "string",
"index": "not_analyzed"
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "float"
}
}
},
"host": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"level": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"logsearch": {
"properties": {
"client-ip": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"logAction": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"maxDurationReached": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"numeric_value": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"searchFinished": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"sessionId": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"site": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"siteId": {
"type": "long"
},
"test": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"timeFromNow": {
"type": "long"
},
"timeFromNowInt": {
"type": "long"
}
}
},
"message": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
}
},
"module": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"path": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"received_from": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"timestamp": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"type": {
"type": "string",
"norms": {
"enabled": false
},
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
}
}
},
"_default_": {
"_all": {
"enabled": true,
"omit_norms": true
},
"dynamic_templates": [
{
"message_field": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": {
"format": "disabled"
},
"type": "string"
},
"match": "message",
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": {
"format": "disabled"
},
"type": "string",
"fields": {
"raw": {
"index": "not_analyzed",
"ignore_above": 256,
"type": "string"
}
}
},
"match": "*",
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "string",
"index": "not_analyzed"
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "float"
}
}
}
}
}
}
},
"kibana-int": {
"mappings": {
"config": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"buildNum": {
"type": "string",
"index": "not_analyzed"
},
"defaultIndex": {
"type": "string"
}
}
},
"index-pattern": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"fieldFormatMap": {
"type": "string"
},
"fields": {
"type": "string"
},
"intervalName": {
"type": "string"
},
"notExpandable": {
"type": "boolean"
},
"timeFieldName": {
"type": "string"
},
"title": {
"type": "string"
}
}
},
"search": {
"properties": {
"columns": {
"type": "string"
},
"description": {
"type": "string"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "string"
}
}
},
"sort": {
"type": "string"
},
"title": {
"type": "string"
},
"version": {
"type": "integer"
}
}
},
"visualization": {
"properties": {
"description": {
"type": "string"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "string"
}
}
},
"savedSearchId": {
"type": "string"
},
"title": {
"type": "string"
},
"uiStateJSON": {
"type": "string"
},
"version": {
"type": "integer"
},
"visState": {
"type": "string"
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "string"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "string"
}
}
},
"optionsJSON": {
"type": "string"
},
"panelsJSON": {
"type": "string"
},
"timeFrom": {
"type": "string"
},
"timeRestore": {
"type": "boolean"
},
"timeTo": {
"type": "string"
},
"title": {
"type": "string"
},
"uiStateJSON": {
"type": "string"
},
"version": {
"type": "integer"
}
}
}
}
}
the template
{
"logstash": {
"order": 0,
"template": "logstash-*",
"settings": {
"index": {
"refresh_interval": "5s"
}
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"message_field": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": {
"format": "disabled"
},
"type": "string"
},
"match_mapping_type": "string",
"match": "message"
}
},
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": {
"format": "disabled"
},
"type": "string",
"fields": {
"raw": {
"index": "not_analyzed",
"ignore_above": 256,
"type": "string"
}
}
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"geoip": {
"dynamic": true,
"properties": {
"location": {
"type": "geo_point"
},
"longitude": {
"type": "float"
},
"latitude": {
"type": "float"
},
"ip": {
"type": "ip"
}
}
},
"@version": {
"index": "not_analyzed",
"type": "string"
}
},
"_all": {
"enabled": true,
"omit_norms": true
}
}
},
"aliases": {}
}
}