Resolve "Provide scripts to just insert bash command history or to just insert kypo portal events" (f9cc5a3f) · Commits · CYBERSEC / ELK LA / elk-portal-commands-events

README.md

+15 −3

Original line number	Diff line number	Diff line
		@@ -27,7 +27,7 @@ On the successful startup (in approximately 30 seconds, in the case, that you ha
		## Library Usage
		Just place the data from selected KYPO event (portal events and bash actions), e.g., `echo - Kobylka 3302`. Then go to the folder with `insert-events.sh` file and run the `insert-events.sh` script as follows (please read the next instructions about file naming convention for audit events):
		```
		$ ./insert-events.sh "PATH-To-YOUR-KYPO-PORTAL-EVENTS-FOLDER" "PATH-To-YOUR-KYPO-PORTAL-LOGS-DIR"
		$ ./insert-events.sh "PATH-To-YOUR-KYPO-PORTAL-EVENTS-FOLDER" "PATH-To-YOUR-KYPO-PORTAL-BASH-HISTORY-COMMANDS-DIR"
		```

		e.g.,
		@@ -40,12 +40,24 @@ Where the content inside `echo - Kobylka 3302 is` as follows:

		![kypo-events-folder](https://gitlab.fi.muni.cz/kypolab/elk-portal-events/-/wikis/uploads/fbebce89ba112084a6ccd9989440f51f/echo_Kobylka_content.PNG)

		and `logs_dir` is as follows:
		and `BASH-HISTORY-COMMANDS-DIR` is as follows:

		![kypo-bash-history-logs-folder](https://gitlab.fi.muni.cz/kypolab/elk-portal-events/-/wikis/uploads/f81e3ddc1031d847bd94af350e48eba8/logs_dir-content.PNG)
		![kypo-bash-history-logs-folder](https://gitlab.fi.muni.cz/kypolab/elk-portal-events/-/wikis/uploads/841b46098f7b1d42a2b6812b0148875a/image.png)

		This script basically goes through all the files in the given directories and inside it search for files that match the pattern `-events.json` or `-useractions.json` and inserts all the content of those files under the correct index into local Elasticsearch instance running on port 9200.


		### Inserting just kypo portal events or kypo bash history commands
		If you want to insert just `kypo portal events` run the script as follows:

		```
		$ ./insert-events.sh "PATH-To-YOUR-KYPO-PORTAL-EVENTS-FOLDER" ""
		```
		Otherwise if you want to insert just `kypo bash history commands` run the script as follows:
		```
		$ ./insert-events.sh "" "PATH-To-YOUR-KYPO-PORTAL-BASH-HISTORY-COMMANDS-DIR"
		```

		## Checking the Data in ELK

		Visit Kibana user interface ran from the previous step on the page `http://localhost:5601/`.

etc/test/kypo-bash-history-logstash-proposal-seda.conf

deleted100644 → 0

+0 −74

Original line number	Diff line number	Diff line
		input {
		http {
		port => 9603
		codec => "json"
		}
		}
		filter {
		# filter messages on only messages which are from bash_history programname and contains KYPO_BASH_ACTION_AUDIT in the message
		if ("KYPO_BASH_ACTION_AUDIT " not in [message]) {drop{}}
		# match json object data which represents the audit event object and save them to auditmessage field
		grok {
		match => { "message" => "%{DATA:data} KYPO_BASH_ACTION_AUDIT --- %{GREEDYDATA:auditmessage}" }
		}
		json {
		source => "auditmessage"
		target => "message"
		}
		# retrieve elements from json to compose Elasticsearch index correctly
		mutate {
		add_field => { "[@metadata][sandboxid]" => "%{[message][sandbox_id]}" }
		}
		# add all Syslog fields to syslog object
		mutate {
		add_field => { "[syslog][procid]" => "%{[message][procid]}"}
		add_field => { "[syslog][facility]" => "%{[message][facility]}"}
		add_field => { "[syslog][severity]" => "%{[message][severity]}"}
		add_field => { "[syslog][@version]" => "%{@version}"}
		add_field => { "[syslog][@timestamp]" => "%{@timestamp}"}
		}
		# here, we eliminate top-level field message and set all nested fields of the message as top-level fields
		ruby {
		code => "
		begin
		message= event.get('message')
		if message!= NIL
		message.keys.each{\|key\|
		event.set(key, message[key])
		}
		event.remove('message')
		end
		end
		"
		}
		# remove command line trailing white spaces
		mutate {
		strip => ["cmd"]
		}
		# copy cmd field to the new cmd_tmp to make further processing on it
		mutate {
		copy => { "cmd" => "cmd_tmp" }
		}
		# this needs to be updated since we could not parse it based on the white space..
		mutate {
		split => ["cmd_tmp", " "]
		add_field => { "cmd_keyword" => "%{[cmd_tmp][0]}" }
		}
		mutate {
		remove_field => [ "[cmd_tmp][0]" ]
		}
		mutate {
		copy => { "cmd_tmp" => "cmd_parameters" }
		}
		# remove unnecessary fields
		mutate {
		remove_field => [ "auditmessage", "cmd_tmp", "procid", "facility", "severity", "@version", "@timestamp"]
		}
		}
		output {
		elasticsearch {
		hosts => [ "elasticsearch:9200" ]
		index => "kypo.bash.logs.sandbox=%{[@metadata][sandboxid]}"
		codec => json
		}
		}
		No newline at end of file

etc/test/kypo-logstash-prod.conf

deleted100644 → 0

+0 −126

Original line number	Diff line number	Diff line
		input {
		tcp {
		host => "127.0.0.1"
		port => 10514
		codec => "json"
		type => "rsyslog"
		}
		}
		filter {
		if [programname] != "kypo2-training" and [programname] != "msf4-history" and [programname] != "command" {drop{}}
		# filter messages on only messages which are from bash_history or kypo portal audit events
		if ("KYPO_BASH_ACTION_AUDIT " in [message]) {
		# match json object data which represents the audit event object and save them to auditmessage field
		grok {
		match => { "message" => "%{DATA:data} KYPO_BASH_ACTION_AUDIT --- %{GREEDYDATA:auditmessage}" }
		}
		json {
		source => "auditmessage"
		target => "message"
		}
		# retrieve elements from json to compose Elasticsearch index correctly
		mutate {
		add_field => { "[@metadata][esindex]" => "kypo.bash.logs.pool=%{[message][pool_id]}.sandbox=%{[message][sandbox_id]}" }
		}
		# add all Syslog fields to syslog object
		mutate {
		add_field => { "[syslog][procid]" => "%{[message][procid]}"}
		add_field => { "[syslog][facility]" => "%{[message][facility]}"}
		add_field => { "[syslog][severity]" => "%{[message][severity]}"}
		add_field => { "[syslog][@version]" => "%{@version}"}
		add_field => { "[syslog][@timestamp]" => "%{@timestamp}"}
		}
		# here, we eliminate top-level field message and set all nested fields of the message as top-level fields
		ruby {
		code => "
		begin
		message= event.get('message')
		if message!= NIL
		message.keys.each{\|key\|
		event.set(key, message[key])
		}
		event.remove('message')
		end
		end
		"
		}
		# remove command line trailing white spaces
		mutate {
		strip => ["cmd"]
		}
		# copy cmd field to the new cmd_tmp to make further processing on it
		mutate {
		copy => { "cmd" => "cmd_tmp" }
		}
		# this needs to be updated since we could not parse it based on the white space..
		mutate {
		split => ["cmd_tmp", " "]
		add_field => { "cmd_keyword" => "%{[cmd_tmp][0]}" }
		}
		mutate {
		remove_field => [ "[cmd_tmp][0]" ]
		}
		mutate {
		copy => { "cmd_tmp" => "cmd_parameters" }
		}
		# remove unnecessary fields
		mutate {
		remove_field => [ "auditmessage", "cmd_tmp", "procid", "facility", "severity", "@version", "@timestamp"]
		}

		} else if ("KYPO_PORTAL_EVENT_AUDIT " in [message]) {
		# match json object data which represents the audit event object and save them to auditmessage field
		grok {
		match => { "message" => "AUDIT %{DATA:data} --- %{GREEDYDATA:auditmessage}" }
		}
		json {
		source => "auditmessage"
		target => "message"
		}
		# retrieve elements from json to compose Elasticsearch index correctly
		mutate {
		add_field => { "[@metadata][esindex]" => "kypo.%{[message][type]}_evt.definition=%{[message][training_definition_id]}.instance=%{[message][training_instance_id]}.run=%{[message][training_run_id]}"}
		}
		# index in Elasticsearch must be lowercase, so we need to lowercase it
		mutate {
		lowercase => [ "[@metadata][eseventtype]" ]
		}
		# add all Syslog fields to syslog object
		mutate {
		add_field => { "[syslog][programname]" => "%{programname}"}
		add_field => { "[syslog][sysloghost]" => "%{sysloghost}"}
		add_field => { "[syslog][procid]" => "%{procid}"}
		add_field => { "[syslog][host]" => "%{host}"}
		add_field => { "[syslog][facility]" => "%{facility}"}
		add_field => { "[syslog][type]" => "%{type}"}
		add_field => { "[syslog][@version]" => "%{@version}"}
		add_field => { "[syslog][severity]" => "%{severity}"}
		add_field => { "[syslog][@timestamp]" => "%{@timestamp}"}
		}
		# remove Syslog fields (these are included in syslog object now)
		mutate {
		remove_field => [ "data", "auditmessage", "programname", "sysloghost", "procid", "host", "facility", "type", "@version", "severity", "@timestamp" ]
		}
		# here, we eliminate top-level field message and set all nested fields of the message as top-level fields
		ruby {
		code => "
		begin
		message= event.get('message')
		if message!= NIL
		message.keys.each{\|key\|
		event.set(key, message[key])
		}
		event.remove('message')
		end
		end
		"
		}
		} else {drop{}}
		}
		output {
		elasticsearch {
		hosts => [ "elasticsearch:9200" ]
		index => "%{[@metadata][esindex]}"
		codec => json
		}
		}
		No newline at end of file

etc/test/kypo-portal-events-old.conf

deleted100644 → 0

+0 −69

Original line number	Diff line number	Diff line
		input {
		tcp {
		host => "127.0.0.1"
		port => 10514
		codec => "json"
		type => "rsyslog"
		}
		}
		filter {
		# filter messages on only messages which are from kypo2-training programname and contains AUDIT in the message
		#if [programname] != "kypo2-training" {drop{}}
		if ("AUDIT " not in [message]) {drop{}}
		# match json object data which represents the audit event object and save them to auditmessage field
		grok {
		match => { "message" => "AUDIT %{DATA:data} --- %{GREEDYDATA:auditmessage}" }
		}
		json {
		source => "auditmessage"
		target => "message"
		}
		# retrieve elements from json to compose Elasticsearch index correctly
		mutate {
		add_field => { "[@metadata][eseventtype]" => "%{[message][type]}" }
		add_field => { "[@metadata][trainingdefinitionid]" => "%{[message][training_definition_id]}"}
		add_field => { "[@metadata][traininginstanceid]" => "%{[message][training_instance_id]}"}
		add_field => { "[@metadata][trainingrunid]" => "%{[message][training_run_id]}"}
		}
		# index in Elasticsearch must be lowercase, so we need to lowercase it
		mutate {
		lowercase => [ "[@metadata][eseventtype]" ]
		}
		# add all Syslog fields to syslog object
		mutate {
		add_field => { "[syslog][programname]" => "%{programname}"}
		add_field => { "[syslog][sysloghost]" => "%{sysloghost}"}
		add_field => { "[syslog][procid]" => "%{procid}"}
		add_field => { "[syslog][host]" => "%{host}"}
		add_field => { "[syslog][facility]" => "%{facility}"}
		add_field => { "[syslog][type]" => "%{type}"}
		add_field => { "[syslog][@version]" => "%{@version}"}
		add_field => { "[syslog][severity]" => "%{severity}"}
		add_field => { "[syslog][@timestamp]" => "%{@timestamp}"}
		}
		# remove Syslog fields (these are included in syslog object now)
		mutate {
		remove_field => [ "data", "auditmessage", "programname", "sysloghost", "procid", "host", "facility", "type", "@version", "severity", "@timestamp" ]
		}
		# here, we eliminate top-level field message and set all nested fields of the message as top-level fields
		ruby {
		code => "
		begin
		message= event.get('message')
		if message!= NIL
		message.keys.each{\|key\|
		event.set(key, message[key])
		}
		event.remove('message')
		end
		end
		"
		}
		}
		output {
		elasticsearch {
		hosts => [ "elasticsearch:9200" ]
		index => "kypo.%{[@metadata][eseventtype]}_evt.definition=%{[@metadata][trainingdefinitionid]}.instance=%{[@metadata][traininginstanceid]}.run=%{[@metadata][trainingrunid]}"
		codec => json
		}
		}
		No newline at end of file

insert-events.sh

+29 −18

Original line number	Diff line number	Diff line
		@@ -9,6 +9,8 @@ TEMPLATE_PATH="template.json"
		curl -H 'Content-Type: application/json' -X PUT -d @${TEMPLATE_PATH} ${ELASTICSEARCH_API_ROOT}${TEMPLATE_INFO}?include_type_name=true
		## POST data to Elastic search
		# KYPO portal events
		if [ ! -z "$1" ]
		then
		cd "$1"
		for FILE in *-events.json
		do
		@@ -18,7 +20,13 @@ do
		curl -X POST -d "$LINE" "${LOGSTASH_API_ROOT_EVENTS}" -H 'Content-Type: application/json'
		done < $FILE
		done
		else
		echo "KYPO PORTAL EVENTS DIRECTORY WAS NOT PROVIDED."
		fi

		# bash history commands
		if [ ! -z "$2" ]
		then
		cd "$2"
		for FILE in *-useractions.json
		do
		@@ -28,3 +36,6 @@ do
		curl -X POST -d "$LINE" "${LOGSTASH_API_ROOT_BASH}"
		done < $FILE
		done
		else
		echo "KYPO BASH COMMANDS DIRECTORY WAS NOT PROVIDED."
		fi