added alot of notebook and docker stuff

logstash config
2018-04-03 10:28:02 +02:00
parent e85d8a9f85
commit a6cc17b436
11 changed files with 310 additions and 0 deletions
--- a/dockerfiles/logstash/configs/syslog.conf
+++ b/dockerfiles/logstash/configs/syslog.conf
@@ -0,0 +1,90 @@
+input {
+  # tcp {
+  #   port => 5000
+  #   type => syslog
+  # }
+  # udp {
+  #   port => 5000
+  #   type => syslog
+  # }
+    file{
+        type => syslog
+        path => "/data/*"
+        start_position => "beginning"
+    }
+}
+filter {
+    if [type] == "syslog" {
+        syslog_pri{}
+        grok {
+            match => { "message" => [
+                "%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}",
+                "%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{GREEDYDATA:syslog_message}"
+                ]
+             }
+            add_field => [ "received_at", "%{@timestamp}" ]
+            add_field => [ "received_from", "%{host}" ]
+        }
+        date {
+            match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+        }
+        mutate {
+    	      # replace => [ "message", "%{syslog_message}" ]
+    	      rename => [ "syslog_message", "message" ]
+    	}
+    }
+    if [syslog_program] == "mwg"{
+        kv{
+            field_split => "|"
+            transform_key => "lowercase"
+            trim_value => '"'
+            target => "mwg"
+            add_tag => "mwg"
+            remove_field => "message"
+            include_keys => [
+            "devtime",
+            "referer",
+            "httpmethod",
+            "bytes",
+            "mediatype",
+            "useragent",
+            "blockreason",
+            "url",
+            "dst",
+            "httpstatus",
+            "src",
+            "urlcategories",
+            "usrname"
+            ]
+        }
+        mutate{
+            convert => { "mwg[bytes]" => "integer" }
+        }
+        date{
+            match => ["mwg[devtime]" , "UNIX_MS"]
+            # target => "mwg[timestamp]"
+        }
+        useragent {
+            source => "mwg[useragent]"
+            target => "mwg[ua]"
+        }
+        geoip {
+            source => "mwg[dst]"
+            target => "mwg[dst_geo]"
+        }
+        # geoip {
+        #     source => "mwg[src]"
+        #     target => "mwg[src_geo]"
+        # }
+
+
+          # mutate {
+          #    split => { "syslog_message" => "|" }
+          # }
+
+    }
+}
+output {
+  elasticsearch { hosts => ["elasticsearch:9200"] }
+  # stdout { codec => rubydebug }
+}