irt/gists
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

added alot of notebook and docker stuff

Browse Source 
logstash config
This commit is contained in:
root
2018-04-03 10:28:02 +02:00
parent e85d8a9f85
commit a6cc17b436
11 changed files with 310 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
dockerfiles/logstash/configs/as_dns.conf Normal file
View File

@@ -0,0 +1,32 @@
input {
# tcp {
# port => 5000
# type => syslog
# }
# udp {
# port => 5000
# type => syslog
# }
file{
type => as_csv
path => "/tmp/logfile"
# path => "/data/*"
start_position => "beginning"
}
}
filter {
csv{
columns => ["endTime","Name","requestUrl","sourceAddress","destinationAddress","priority","Device Vendor","Device Product","Non-CEF Raw Message"]
}
grok { match => { "message" => ["%{DATESTAMP:datum}"] } }
date { match => [ "datum","yy/MM/dd HH:mm:ss" ] timezone => "CET" }
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "arcsight"
# index => "arcsight-%{+YYYY.MM.dd}"
}
# stdout { codec => rubydebug }
}

90
dockerfiles/logstash/configs/bwi.conf Normal file
View File

@@ -0,0 +1,90 @@
input {
# tcp {
# port => 5000
# type => syslog
# }
# udp {
# port => 5000
# type => syslog
# }
file{
type => syslog
path => "/data/*"
start_position => "beginning"
}
}
filter {
if [type] == "syslog" {
syslog_pri{}
grok {
match => { "message" => [
"%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}",
"%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{GREEDYDATA:syslog_message}"
]
}
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
mutate {
# replace => [ "message", "%{syslog_message}" ]
rename => [ "syslog_message", "message" ]
}
}
if [syslog_program] == "mwg"{
kv{
field_split => "|"
transform_key => "lowercase"
trim_value => '"'
target => "mwg"
add_tag => "mwg"
remove_field => "message"
include_keys => [
"devtime",
"referer",
"httpmethod",
"bytes",
"mediatype",
"useragent",
"blockreason",
"url",
"dst",
"httpstatus",
"src",
"urlcategories",
"usrname"
]
}
mutate{
convert => { "mwg[bytes]" => "integer" }
}
date{
match => ["mwg[devtime]" , "UNIX_MS"]
# target => "mwg[timestamp]"
}
useragent {
source => "mwg[useragent]"
target => "mwg[ua]"
}
geoip {
source => "mwg[dst]"
target => "mwg[dst_geo]"
}
# geoip {
# source => "mwg[src]"
# target => "mwg[src_geo]"
# }
# mutate {
# split => { "syslog_message" => "|" }
# }
}
}
output {
elasticsearch { hosts => ["elasticsearch:9200"] }
# stdout { codec => rubydebug }
}

28
dockerfiles/logstash/configs/first.conf Normal file
View File

@@ -0,0 +1,28 @@
input {
tcp {
port => 514
type => syslog
}
udp {
port => 514
type => syslog
}
}
filter {
if [type] == "syslog" {
grok {
match => { "message" => "(<%{POSINT:syslog_pri}>|)%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
output {
elasticsearch { hosts => ["elasticsearch:9200"] }
stdout { codec => rubydebug }
}

90
dockerfiles/logstash/configs/syslog.conf Normal file
View File

@@ -0,0 +1,90 @@
input {
# tcp {
# port => 5000
# type => syslog
# }
# udp {
# port => 5000
# type => syslog
# }
file{
type => syslog
path => "/data/*"
start_position => "beginning"
}
}
filter {
if [type] == "syslog" {
syslog_pri{}
grok {
match => { "message" => [
"%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}",
"%{SYSLOGTIMESTAMP:syslog_timestamp} (%{SYSLOGHOST:syslog_hostname} |)%{GREEDYDATA:syslog_message}"
]
}
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
mutate {
# replace => [ "message", "%{syslog_message}" ]
rename => [ "syslog_message", "message" ]
}
}
if [syslog_program] == "mwg"{
kv{
field_split => "|"
transform_key => "lowercase"
trim_value => '"'
target => "mwg"
add_tag => "mwg"
remove_field => "message"
include_keys => [
"devtime",
"referer",
"httpmethod",
"bytes",
"mediatype",
"useragent",
"blockreason",
"url",
"dst",
"httpstatus",
"src",
"urlcategories",
"usrname"
]
}
mutate{
convert => { "mwg[bytes]" => "integer" }
}
date{
match => ["mwg[devtime]" , "UNIX_MS"]
# target => "mwg[timestamp]"
}
useragent {
source => "mwg[useragent]"
target => "mwg[ua]"
}
geoip {
source => "mwg[dst]"
target => "mwg[dst_geo]"
}
# geoip {
# source => "mwg[src]"
# target => "mwg[src_geo]"
# }
# mutate {
# split => { "syslog_message" => "|" }
# }
}
}
output {
elasticsearch { hosts => ["elasticsearch:9200"] }
# stdout { codec => rubydebug }
}