irt/gists
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

visidata: enhance IOC plugins with improved lookups and validation

Browse Source 
Expand iplib, iptype, and ioc plugins with better caching, throttling,
and lookup logic. Update validation script and showcase journal accordingly.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
tobias
2026-03-07 22:49:49 +01:00
parent 559fa38c04
commit 49db614262
6 changed files with 608 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
config/visidata/plugins/iplib.py
View File

@@ -8,7 +8,7 @@ any Python interpreter.
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Dict, Optional, Tuple
from typing import Any, Dict, Iterable, List, Optional, Tuple
class JSONNode:
@@ -124,11 +124,33 @@ class ASNInfo:
@dataclass(frozen=True)
class VTInfo:
object_type: str = ""
malicious: int = 0
suspicious: int = 0
harmless: int = 0
undetected: int = 0
timeout: int = 0
failure: int = 0
confirmed_timeout: int = 0
type_unsupported: int = 0
total: int = 0
reputation: Optional[int] = None
votes_harmless: int = 0
votes_malicious: int = 0
score: Optional[float] = None
categories: Tuple[str, ...] = ()
tags: Tuple[str, ...] = ()
names: Tuple[str, ...] = ()
name: str = ""
asn: str = ""
as_owner: str = ""
continent: str = ""
country: str = ""
network: str = ""
ip: str = ""
ips: Tuple[str, ...] = ()
last_analysis_date: Optional[int] = None
last_modification_date: Optional[int] = None
raw: Optional[Dict[str, Any]] = None
source: str = ""
@@ -136,6 +158,19 @@ class VTInfo:
def data(self) -> JSONNode:
return JSONNode(self.raw)
@property
def attrs(self) -> JSONNode:
attrs = ((self.raw or {}).get("data") or {}).get("attributes") or {}
return JSONNode(attrs)
@property
def results(self) -> JSONNode:
return self.attrs.get("last_analysis_results", {})
@property
def stats(self) -> JSONNode:
return self.attrs.get("last_analysis_stats", {})
@property
def verdict(self) -> str:
return "" if self.total <= 0 else f"{self.malicious}/{self.total}"
@@ -144,6 +179,10 @@ class VTInfo:
def ratio(self) -> Optional[float]:
return None if self.total <= 0 else self.malicious / self.total
@property
def confidence(self) -> Optional[float]:
return self.score
@property
def type(self) -> str: # noqa: A003
return self.category
@@ -153,6 +192,9 @@ class VTInfo:
return self.categories[0] if self.categories else ""
def __getattr__(self, name: str) -> Any:
attrs = ((self.raw or {}).get("data") or {}).get("attributes") or {}
if isinstance(attrs, dict) and name in attrs:
return attrs.get(name)
if isinstance(self.raw, dict) and name in self.raw:
return self.raw.get(name)
raise AttributeError(name)
@@ -317,42 +359,208 @@ def parse_geo_maxmind(raw: Optional[Dict[str, Any]]) -> GeoInfo:
def parse_vt_ip(data: Optional[Dict[str, Any]]) -> VTInfo:
return _parse_vt(data, object_type="ip")
def parse_vt_domain(data: Optional[Dict[str, Any]]) -> VTInfo:
return _parse_vt(data, object_type="domain")
def parse_vt_url(data: Optional[Dict[str, Any]]) -> VTInfo:
return _parse_vt(data, object_type="url")
def parse_vt_file(data: Optional[Dict[str, Any]]) -> VTInfo:
return _parse_vt(data, object_type="file")
def _safe_int(v: Any, default: Optional[int] = 0) -> Optional[int]:
try:
return int(v)
except Exception:
return default
def _uniq(items: Iterable[str]) -> Tuple[str, ...]:
out: List[str] = []
seen = set()
for x in items:
s = str(x or "").strip()
if not s or s in seen:
continue
seen.add(s)
out.append(s)
return tuple(out)
def _extract_categories(attrs: Dict[str, Any]) -> Tuple[str, ...]:
cats = attrs.get("categories") or {}
if isinstance(cats, dict):
return _uniq(str(v) for v in cats.values() if v)
if isinstance(cats, list):
return _uniq(str(v) for v in cats if v)
return ()
def _extract_names(attrs: Dict[str, Any], object_type: str) -> Tuple[str, ...]:
names: List[str] = []
threat_names = attrs.get("threat_names")
if isinstance(threat_names, list):
names.extend(str(x) for x in threat_names if x)
elif isinstance(threat_names, str) and threat_names:
names.append(threat_names)
ptc = attrs.get("popular_threat_classification") or {}
if isinstance(ptc, dict):
rows = ptc.get("popular_threat_name")
if isinstance(rows, list):
scored = []
for row in rows:
if not isinstance(row, dict):
continue
val = str(row.get("value") or "").strip()
if not val:
continue
scored.append((_safe_int(row.get("count"), 0) or 0, val))
scored.sort(reverse=True)
names.extend(v for _, v in scored)
suggested = str(ptc.get("suggested_threat_label") or "").strip()
if suggested:
names.append(suggested)
verdicts = attrs.get("sandbox_verdicts") or {}
if isinstance(verdicts, dict):
for v in verdicts.values():
if not isinstance(v, dict):
continue
mn = v.get("malware_names")
if isinstance(mn, list):
names.extend(str(x) for x in mn if x)
mc = v.get("malware_classification")
if isinstance(mc, list):
names.extend(str(x) for x in mc if x)
if object_type == "file":
meaningful = str(attrs.get("meaningful_name") or "").strip()
if meaningful:
names.append(meaningful)
return _uniq(names)
def _extract_ips(attrs: Dict[str, Any]) -> Tuple[str, ...]:
ips: List[str] = []
records = attrs.get("last_dns_records")
if isinstance(records, list):
for rec in records:
if not isinstance(rec, dict):
continue
rtype = str(rec.get("type") or "").upper()
if rtype not in ("A", "AAAA"):
continue
val = str(rec.get("value") or "").strip()
if val:
ips.append(val)
lserv = attrs.get("last_serving_ip_address")
if isinstance(lserv, dict):
val = str(lserv.get("id") or lserv.get("value") or "").strip()
if val:
ips.append(val)
elif isinstance(lserv, str) and lserv:
ips.append(lserv)
return _uniq(ips)
def _parse_vt(data: Optional[Dict[str, Any]], *, object_type: str) -> VTInfo:
data = data or {}
attrs = (((data.get("data") or {}).get("attributes") or {}) if isinstance(data, dict) else {})
attrs = (
((data.get("data") or {}).get("attributes") or {})
if isinstance(data, dict)
else {}
)
stats = attrs.get("last_analysis_stats") or {}
malicious = _safe_int(stats.get("malicious"), 0) or 0
suspicious = _safe_int(stats.get("suspicious"), 0) or 0
harmless = _safe_int(stats.get("harmless"), 0) or 0
undetected = _safe_int(stats.get("undetected"), 0) or 0
timeout = _safe_int(stats.get("timeout"), 0) or 0
failure = _safe_int(stats.get("failure"), 0) or 0
confirmed_timeout = _safe_int(stats.get("confirmed-timeout"), 0) or 0
type_unsupported = _safe_int(stats.get("type-unsupported"), 0) or 0
try:
malicious = int(stats.get("malicious") or 0)
except Exception:
malicious = 0
try:
suspicious = int(stats.get("suspicious") or 0)
except Exception:
suspicious = 0
try:
total = int(sum(int(v or 0) for v in stats.values()))
total = int(sum(_safe_int(v, 0) or 0 for v in stats.values()))
except Exception:
total = 0
cats = attrs.get("categories") or {}
if isinstance(cats, dict):
categories = tuple(str(v) for v in cats.values() if v)
elif isinstance(cats, list):
categories = tuple(str(v) for v in cats if v)
else:
categories = ()
categories = _extract_categories(attrs)
names = _extract_names(attrs, object_type=object_type)
tags = _uniq(str(x) for x in (attrs.get("tags") or []) if x)
ips = _extract_ips(attrs)
rep = attrs.get("reputation")
try:
reputation = int(rep) if rep is not None else None
except Exception:
reputation = None
rep = _safe_int(attrs.get("reputation"), None)
votes = attrs.get("total_votes") or {}
votes_harmless = 0
votes_malicious = 0
if isinstance(votes, dict):
votes_harmless = _safe_int(votes.get("harmless"), 0) or 0
votes_malicious = _safe_int(votes.get("malicious"), 0) or 0
asn_raw = attrs.get("asn")
asn = ""
if asn_raw is not None:
sval = str(asn_raw).strip()
if sval:
asn = sval if sval.upper().startswith("AS") else f"AS{sval}"
ip = ""
for x in ips:
if "." in x:
ip = x
break
if not ip and ips:
ip = ips[0]
score = None if total <= 0 else malicious / total
name = names[0] if names else ""
la = _safe_int(attrs.get("last_analysis_date"), None)
lm = _safe_int(attrs.get("last_modification_date"), None)
return VTInfo(
object_type=object_type,
malicious=malicious,
suspicious=suspicious,
harmless=harmless,
undetected=undetected,
timeout=timeout,
failure=failure,
confirmed_timeout=confirmed_timeout,
type_unsupported=type_unsupported,
total=total,
reputation=reputation,
reputation=rep,
votes_harmless=votes_harmless,
votes_malicious=votes_malicious,
score=score,
categories=categories,
tags=tags,
names=names,
name=name,
asn=asn,
as_owner=str(attrs.get("as_owner") or ""),
continent=str(attrs.get("continent") or ""),
country=str(attrs.get("country") or ""),
network=str(attrs.get("network") or ""),
ip=ip,
ips=ips,
last_analysis_date=la,
last_modification_date=lm,
raw=data,
source="virustotal",
)