pfsense2jsonl/pfjson2cim.go

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
)

func main() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := scanner.Text()
		if len(strings.TrimSpace(line)) == 0 {
			continue
		}

		// Unmarshal the JSON event
		var event map[string]interface{}
		if err := json.Unmarshal([]byte(line), &event); err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing JSON: %v\n", err)
			continue
		}

		// Transform the event into a flat CIM structure.
		cimEvent := transformToCIM(event)

		// Marshal the new event and output it
		out, err := json.Marshal(cimEvent)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error marshaling CIM event: %v\n", err)
			continue
		}
		fmt.Println(string(out))
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "Error reading input: %v\n", err)
		os.Exit(1)
	}
}

// transformToCIM maps the nested pfSense log structure to a flat CIM structure.
func transformToCIM(event map[string]interface{}) map[string]interface{} {
	cim := make(map[string]interface{})

	// Timestamp: preserve it as is.
	if ts, ok := event["timestamp"]; ok {
		cim["timestamp"] = ts
	}

	// Optionally copy log_level if available.
	if lvl, ok := event["log_level"]; ok {
		cim["log_level"] = lvl
	}

	// Our pfSense log is expected to have a "log" object.
	logObj, ok := event["log"].(map[string]interface{})
	if !ok {
		// If not, return the original event.
		return event
	}

	// Extract common fields from "base"
	base, _ := logObj["base"].(map[string]interface{})
	if base != nil {
		if act, ok := base["action"].(string); ok && act != "" {
			cim["action"] = strings.ToLower(act)
		}
		if dir, ok := base["direction"].(string); ok && dir != "" {
			cim["direction"] = strings.ToLower(dir)
		}
		if rn, ok := base["rule_number"]; ok {
			// try converting rule_number to integer if possible.
			if rnStr, ok := rn.(string); ok {
				if rnInt, err := strconv.Atoi(rnStr); err == nil {
					cim["rule_number"] = rnInt
				} else {
					cim["rule_number"] = rnStr
				}
			} else {
				cim["rule_number"] = rn
			}
		}
		if reason, ok := base["reason"].(string); ok && reason != "" {
			cim["reason"] = reason
		}
	}

	// Extract IP data: source, destination, and packet length
	ipData, _ := logObj["ip_data"].(map[string]interface{})
	if ipData != nil {
		if src, ok := ipData["source_address"].(string); ok && src != "" {
			cim["src"] = src
		}
		if dest, ok := ipData["destination_address"].(string); ok && dest != "" {
			cim["dest"] = dest
		}
		if length, ok := ipData["length"].(string); ok && length != "" {
			if l, err := strconv.Atoi(length); err == nil {
				cim["packet_length"] = l
			} else {
				cim["packet_length"] = length
			}
		}
	}

	// Determine transport protocol from ip_specific_data.
	ipSpec, _ := logObj["ip_specific_data"].(map[string]interface{})
	var protocol string
	if ipSpec != nil {
		if ipVersion, ok := base["ip_version"].(string); ok {
			if ipVersion == "4" {
				if ipv4, ok := ipSpec["ipv4_header"].(map[string]interface{}); ok {
					if pt, ok := ipv4["protocol_text"].(string); ok {
						protocol = strings.ToLower(pt)
					}
				}
			} else if ipVersion == "6" {
				if ipv6, ok := ipSpec["ipv6_header"].(map[string]interface{}); ok {
					if pt, ok := ipv6["protocol_text"].(string); ok {
						protocol = strings.ToLower(pt)
					}
				}
			}
		}
	}
	if protocol != "" {
		cim["transport"] = protocol
	}

	// Map protocol-specific data for TCP and UDP.
	protoSpec, _ := logObj["protocol_specific_data"].(map[string]interface{})
	if protoSpec != nil {
		// For TCP/UDP, map ports and data length.
		if protocol == "tcp" || protocol == "udp" {
			if sp, ok := protoSpec["source_port"].(string); ok && sp != "" {
				if port, err := strconv.Atoi(sp); err == nil {
					cim["src_port"] = port
				} else {
					cim["src_port"] = sp
				}
			}
			if dp, ok := protoSpec["destination_port"].(string); ok && dp != "" {
				if port, err := strconv.Atoi(dp); err == nil {
					cim["dest_port"] = port
				} else {
					cim["dest_port"] = dp
				}
			}
			if dl, ok := protoSpec["data_length"].(string); ok && dl != "" {
				if d, err := strconv.Atoi(dl); err == nil {
					// Map to bytes_in or bytes_out based on direction.
					if dstr, ok := cim["direction"].(string); ok {
						if dstr == "in" {
							cim["bytes_in"] = d
						} else if dstr == "out" {
							cim["bytes_out"] = d
						} else {
							cim["data_length"] = d
						}
					} else {
						cim["data_length"] = d
					}
				} else {
					cim["data_length"] = dl
				}
			}
		} else {
			// For non-TCP/UDP protocols, if ports exist in the raw data, include them.
			if sp, ok := protoSpec["source_port"].(string); ok && sp != "" {
				cim["src_port"] = sp
			}
			if dp, ok := protoSpec["destination_port"].(string); ok && dp != "" {
				cim["dest_port"] = dp
			}
		}
	}

	// Return the new flat event with only CIM fields.
	return cim
}