visidata: add replayable IOC showcase and usage docs

Provide a sample dataset and cmdlog that exercise typed IOC enrichment while keeping heavy lookups scoped for practical throttled runs, and document how to run it.

config/visidata/README.md

@@ -20,6 +20,36 @@ On VisiData 3.3, `$VD_DIR` defaults to:
 
 Plugins are installed into `$VD_DIR/plugins/` and imported via the top-level `plugins` package.
 
+## Showcase Demo
+
+This repo includes a self-contained sample dataset + command log to demonstrate the local IOC/IP features:
+
+- `showcase_ioc.tsv` (sample IOC rows)
+- `showcase_ioc.vdj` (replay file)
+
+Run it interactively from this repo root:
+
+```bash
+vd --visidata-dir "$PWD" --config "$PWD/visidatarc" --play showcase_ioc.vdj
+```
+
+What it showcases:
+- custom types: `IP`, `Domain`, `URL`, `Hash`
+- IP membership expressions: `src_ip * network`
+- URL parsing fields: `url.host`, `url.parts.path`, `url.domain`
+- hash classification: `file_hash.kind`
+- IP lookups: `src_ip.ipinfo.*`, `src_ip.asn.*`, `src_ip.geo.*`, `src_ip.country()`
+- provider visibility: `src_ip.geo.source`, `src_ip.asn.source`, `domain.dns.source`
+- domain/network intel: `domain.dns.*`, `domain.rdap.*`
+- hash intel: `file_hash.mb.*` (MalwareBazaar)
+- VirusTotal lookups: `src_ip.vt.*`, `file_hash.vt.*`, `domain.vt.*`, `url.vt.*`
+- local plugin command: `tke-hidecol`
+
+Lookup notes:
+- VT columns require `options.tke_vt_api_key` (or `VT_API_KEY` / `VIRUSTOTAL_API_KEY` / `~/.virustotal_api_key`).
+- IPInfo/ASN/Geo columns use free providers and may be rate-limited; `options.tke_ipinfo_token` improves reliability.
+- To keep replays practical with strict throttling, some heavy lookup columns are intentionally limited to a subset of rows.
+
 ### `plugins/hidecol.py`
 
 Adds a command to hide columns that are empty or constant across all rows.
@@ -121,4 +151,3 @@ Concretely:
 - If you have legacy functions in `visidatarc` that overlap with the new IP lookups, refactor those functions into a shared module (e.g. `plugins/lookups.py`) and have both `visidatarc` and `plugins/iptype.py` call into it.
 
 This keeps backward-compatible names available while ensuring caching/auth/provider behavior is implemented in one place.
-

config/visidata/showcase_ioc.tsv

@@ -0,0 +1,8 @@
+event_id	src_ip	dst_ip	network	domain	url	file_hash	constant	empty_col
+evt-001	8.8.8.8	192.168.1.10	8.8.8.0/24	google.com	https://www.google.com/search?q=visidata	44D88612FEA8A8F36DE82E1278ABB02F	KEEP	
+evt-002	1.1.1.1	10.0.5.9	10.0.0.0/8	cloudflare.com	example.org/download?id=1	3395856CE81F2B7382DEE72602F798B642F14140	KEEP	
+evt-003	2606:4700:4700::1111	172.16.0.5	2606:4700:4700::/48	example.net	http://sub.example.net/path#frag	E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855	KEEP	
+evt-004	192.0.2.15	203.0.113.5	192.0.2.0/24	openai.com	openai.com/research	D41D8CD98F00B204E9800998ECF8427E	KEEP	
+evt-005	bad-ip	203.0.113.250	203.0.113.0/24	test.example	https://test.example:8443/login?id=7	A94A8FE5CCB19BA61C4C0873D391E987982FBBD3	KEEP	
+evt-006	2001:db8::1234	fe80::1	2001:db8::/32	WWW.GitHub.COM.	http://[2001:db8::1]/admin	notahash	KEEP	
+evt-007	8.8.8.8	198.51.100.9	8.8.0.0/16	google.com	https://google.com/	44D88612FEA8A8F36DE82E1278ABB02F	KEEP	

config/visidata/showcase_ioc.vdj

@@ -0,0 +1,37 @@
+#!vd -p
+{"sheet": null, "col": null, "row": null, "longname": "open-file", "input": "showcase_ioc.tsv", "keystrokes": "o", "comment": "Open IOC showcase dataset"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "type-ip", "input": "", "keystrokes": "", "comment": "Set source IP column to custom IP type"}
+{"sheet": "showcase_ioc", "col": "dst_ip", "row": "", "longname": "type-ip", "input": "", "keystrokes": "", "comment": "Set destination IP column to custom IP type"}
+{"sheet": "showcase_ioc", "col": "network", "row": "", "longname": "type-ip", "input": "", "keystrokes": "", "comment": "Set network column to IP/CIDR type"}
+{"sheet": "showcase_ioc", "col": "domain", "row": "", "longname": "type-domain", "input": "", "keystrokes": "", "comment": "Set domain column to Domain type"}
+{"sheet": "showcase_ioc", "col": "url", "row": "", "longname": "type-url-ioc", "input": "", "keystrokes": "", "comment": "Set URL column to IOC URL type"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "type-hash", "input": "", "keystrokes": "", "comment": "Set hash column to IOC Hash type"}
+{"sheet": "showcase_ioc", "col": "constant", "row": "", "longname": "tke-hidecol", "input": "", "keystrokes": "", "comment": "Hide empty and superfluous source columns"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "src_ip * network", "keystrokes": "=", "comment": "IP membership operator on typed values"}
+{"sheet": "showcase_ioc", "col": "url", "row": "", "longname": "addcol-expr", "input": "url.host", "keystrokes": "=", "comment": "Extract parsed URL host via URL type"}
+{"sheet": "showcase_ioc", "col": "url", "row": "", "longname": "addcol-expr", "input": "url.domain", "keystrokes": "=", "comment": "Convert URL host into DomainValue"}
+{"sheet": "showcase_ioc", "col": "url", "row": "", "longname": "addcol-expr", "input": "url.parts.path", "keystrokes": "=", "comment": "Show parsed URL path"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "addcol-expr", "input": "file_hash.kind", "keystrokes": "=", "comment": "Detect MD5/SHA1/SHA256 hash kind"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.ipinfo.country or ''", "keystrokes": "=", "comment": "IPInfo country (limited rows to keep demo fast)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.ipinfo.org or ''", "keystrokes": "=", "comment": "IPInfo org (limited rows to keep demo fast)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.asn.asn or ''", "keystrokes": "=", "comment": "ASN lookup (limited rows to keep demo fast)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.asn.name or ''", "keystrokes": "=", "comment": "ASN name lookup (limited rows to keep demo fast)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.geo.country_code or ''", "keystrokes": "=", "comment": "GeoIP country code (limited rows to keep demo fast)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.geo.city or ''", "keystrokes": "=", "comment": "GeoIP city (limited rows to keep demo fast)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and src_ip and src_ip.vt.verdict or ''", "keystrokes": "=", "comment": "VirusTotal IP verdict (single row for rate-limited API)"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and file_hash and file_hash.vt.verdict or ''", "keystrokes": "=", "comment": "VirusTotal hash verdict (single row for rate-limited API)"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and file_hash and file_hash.vt.malicious or ''", "keystrokes": "=", "comment": "VirusTotal hash malicious count (single row)"}
+{"sheet": "showcase_ioc", "col": "domain", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and domain and domain.vt.verdict or ''", "keystrokes": "=", "comment": "VirusTotal domain verdict (single row)"}
+{"sheet": "showcase_ioc", "col": "url", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and url and url.vt.verdict or ''", "keystrokes": "=", "comment": "VirusTotal URL verdict (single row)"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.country() or ''", "keystrokes": "=", "comment": "Best country helper"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.geo.source or ''", "keystrokes": "=", "comment": "Geo provider source"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and src_ip and src_ip.asn.source or ''", "keystrokes": "=", "comment": "ASN provider source"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and src_ip and src_ip.vt.category or ''", "keystrokes": "=", "comment": "VirusTotal IP category"}
+{"sheet": "showcase_ioc", "col": "src_ip", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and src_ip and src_ip.vt.malicious or ''", "keystrokes": "=", "comment": "VirusTotal IP malicious count"}
+{"sheet": "showcase_ioc", "col": "domain", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and domain and domain.dns.source or ''", "keystrokes": "=", "comment": "DNS lookup source"}
+{"sheet": "showcase_ioc", "col": "domain", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and domain and ','.join(domain.dns.a) or ''", "keystrokes": "=", "comment": "DNS A records"}
+{"sheet": "showcase_ioc", "col": "domain", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and domain and ','.join(domain.dns.mx) or ''", "keystrokes": "=", "comment": "DNS MX records"}
+{"sheet": "showcase_ioc", "col": "domain", "row": "", "longname": "addcol-expr", "input": "event_id in ('evt-001','evt-002') and domain and domain.rdap.objectClassName or ''", "keystrokes": "=", "comment": "RDAP object class"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and file_hash and file_hash.mb.status or ''", "keystrokes": "=", "comment": "MalwareBazaar query status"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and file_hash and file_hash.mb.signature or ''", "keystrokes": "=", "comment": "MalwareBazaar signature"}
+{"sheet": "showcase_ioc", "col": "file_hash", "row": "", "longname": "addcol-expr", "input": "event_id == 'evt-001' and file_hash and ','.join(file_hash.mb.tags) or ''", "keystrokes": "=", "comment": "MalwareBazaar tags"}