From 92c7fc69c67337bda33c02599d0e88836e30d49e Mon Sep 17 00:00:00 2001 From: tobias Date: Sun, 24 Aug 2025 19:50:00 +0200 Subject: [PATCH] Restructure repository: organize tools by purpose, create what search tool - Move single-file tools to tools/ organized by category (security, forensics, data, etc.) - Move multi-file projects to projects/ (go-tools, puzzlebox, timesketch, rust-tools) - Move system scripts to scripts/ (proxy, display, setup, windows) - Organize config files in config/ (shell, visidata, applications) - Move experimental tools to archive/experimental - Create 'what' fuzzy search tool with progressive enhancement (ollama->fzf->grep) - Add initial metadata database for intelligent tool discovery - Preserve git history using 'git mv' commands --- .DS_Store | Bin 0 -> 8196 bytes .what_db.json | 149 ++++++ WARP.md | 141 ++++++ {codegrab => archive/awk}/ips.awk | 0 {codegrab => archive/awk}/map.awk | 0 {codegrab => archive/binaries}/csv_cols | 0 {codegrab => archive/binaries}/mapping | 0 {collected => archive/collected}/README.md | 0 {collected => archive/collected}/appid.tsv | 0 {collected => archive/collected}/commands | 0 {collected => archive/collected}/toollist | 0 {codegrab => archive/experimental}/flm.py | 0 {codegrab => archive/experimental}/fuzz.sh | 0 .../experimental}/hydrogentest.py | 0 {codegrab => archive/experimental}/kv.py | 0 {codegrab => archive/experimental}/lpic.sh | 0 .../experimental}/matplottest.py | 0 codegrab/.DS_Store | Bin 0 -> 6148 bytes codegrab/imphash.go | 1 + config/.DS_Store | Bin 0 -> 6148 bytes .../shell}/agnoster.zsh-theme | 0 config/{ => shell}/bash_aliases | 0 config/{ => shell}/bash_prompt | 0 config/{ => shell}/inputrc | 0 config/{ => shell}/shell_aliases | 0 .../shell}/solarized.dircolors | 0 config/{ => visidata}/access_log.vdj | 0 .../plugins}/hidecol.py | 0 config/{ => visidata}/visidatarc | 0 dockerfiles/.DS_Store | Bin 0 -> 8196 bytes .../go-tools}/go/csv2json/csv2json.go | 0 .../go-tools}/go/gobetween/gobetween.go | 0 {tools => projects/go-tools}/go/goinfo/go.mod | 0 .../go-tools}/go/goinfo/goinfo.go | 0 .../go-tools}/go/goipgrep/ipgrep.go | 0 .../go-tools}/go/gosoft/gosoft.go | 0 .../go-tools}/go/gouniq/gouniq.go | 0 {codegrab => projects}/puzzlebox/solution | 0 {codegrab => projects}/puzzlebox/solve.py | 0 {codegrab => projects}/puzzlebox/solve0.py | 0 {codegrab => projects}/puzzlebox/solve2.py | 0 {codegrab => projects}/puzzlebox/solve_mp.py | 0 {codegrab => projects}/puzzlebox/voxels.py | 0 projects/rust-tools/between.rs | 55 +++ projects/rust-tools/uniq.rs | 41 ++ projects/rust-tools/uniq2.rs | 36 ++ .../timesketch}/deploy_timesketch.sh | 0 restructure_git.sh | 213 +++++++++ scripts/display/reset_screens.sh | 4 + .../display}/toggle_display.sh | 0 .../display}/toggle_touchpad | 0 {systemscripts => scripts}/proxy/get_proxy.sh | 0 .../proxy/update_apt_proxy.sh | 0 .../proxy/update_bashrc_proxy.sh | 0 .../proxy/update_service_proxy.sh | 0 {systemscripts => scripts/setup}/automountctl | 0 {systemscripts => scripts/setup}/share.sh | 0 .../setup}/terminal-logs.sh | 0 .../windows}/Get-ZimmermanTools.ps1 | 0 .../windows}/getscreen.psm1 | 0 {codegrab => scripts/windows}/sbom.ps1 | 0 tools/.DS_Store | Bin 0 -> 6148 bytes {codegrab => tools/cloud}/cloudsend.py | 0 {codegrab => tools/cloud}/cloudsend.sh | 0 {codegrab => tools/cloud}/speech.py | 0 {codegrab => tools/cloud}/vqa3.py | 0 {codegrab => tools/cloud}/youtube_resolve.sh | 0 {codegrab => tools}/ctf/filtertext.py | 0 {codegrab => tools}/ctf/getjs.py | 0 {codegrab => tools}/ctf/guess.py | 0 {codegrab => tools}/ctf/ps_.py | 0 {codegrab => tools}/ctf/search.py | 0 {codegrab => tools}/ctf/submit_flag.sh | 0 {codegrab => tools}/ctf/transpose.py | 0 tools/{ => data}/between | 0 tools/{ => data}/concat.py | 0 tools/{ => data}/csv_get | 0 tools/{ => data}/domgrep.py | 0 tools/{ => data}/geturls.py | 0 {codegrab => tools/data}/json_save.py | 0 {codegrab => tools/data}/kv_parse.py | 0 tools/{ => data}/quickchardet.py | 0 tools/{ => data}/split_linewise.py | 0 {codegrab => tools/data}/uniq.py | 0 tools/{ => data}/unum.py | 0 tools/{ => data}/urldecode.py | 0 {codegrab => tools/data}/vba_chr_decode.py | 0 {codegrab => tools/forensics}/chechsqlite.py | 0 .../forensics}/extractfolder.py | 0 {codegrab => tools/forensics}/process_leak.py | 0 {codegrab => tools/formats}/convert2pdf.sh | 0 {codegrab => tools/formats}/flatpdf.sh | 0 tools/{ => formats}/rename.mime.py | 0 {codegrab => tools/hashing}/hashzip.py | 0 tools/{ => hashing}/libarchivesum.py | 0 tools/{ => hashing}/scatterhash.py | 0 tools/{ => hashing}/tarsum.py | 0 {codegrab => tools/network}/fritzshark.sh | 0 {codegrab => tools/network}/fritzshark2.sh | 0 tools/{ => network}/get_ntp.py | 0 tools/{ => network}/get_stp.sh | 0 tools/{ => network}/ipgrep | 0 {codegrab => tools/security}/certwipe | 0 {codegrab => tools/security}/imphash.py | 0 {codegrab => tools/security}/scan_vt.py | 0 {codegrab => tools/security}/scapy_arp.py | 0 .../security}/simple_portscan.py | 0 {codegrab => tools/security}/smtpbanner.py | 0 {codegrab => tools/security}/testpw.py | 0 {codegrab => tools/security}/vt_download.py | 0 {codegrab => tools/security}/vt_ip.py | 0 {codegrab => tools/security}/vt_pdns.py | 0 tools/{ => system}/backup_docker.sh | 0 {codegrab => tools/system}/ltop.py | 0 tools/{ => system}/restore_docker.sh | 0 tools/{ => system}/watchgrowth.sh | 0 {codegrab => tools/system}/wipe.sh | 0 {codegrab => tools/text}/depth | 0 {codegrab => tools/text}/probability.py | 0 what | 423 ++++++++++++++++++ 120 files changed, 1063 insertions(+) create mode 100644 .DS_Store create mode 100644 .what_db.json create mode 100644 WARP.md rename {codegrab => archive/awk}/ips.awk (100%) rename {codegrab => archive/awk}/map.awk (100%) rename {codegrab => archive/binaries}/csv_cols (100%) rename {codegrab => archive/binaries}/mapping (100%) rename {collected => archive/collected}/README.md (100%) rename {collected => archive/collected}/appid.tsv (100%) rename {collected => archive/collected}/commands (100%) rename {collected => archive/collected}/toollist (100%) rename {codegrab => archive/experimental}/flm.py (100%) rename {codegrab => archive/experimental}/fuzz.sh (100%) rename {codegrab => archive/experimental}/hydrogentest.py (100%) rename {codegrab => archive/experimental}/kv.py (100%) rename {codegrab => archive/experimental}/lpic.sh (100%) rename {codegrab => archive/experimental}/matplottest.py (100%) create mode 100644 codegrab/.DS_Store create mode 100644 codegrab/imphash.go create mode 100644 config/.DS_Store rename {systemscripts => config/shell}/agnoster.zsh-theme (100%) rename config/{ => shell}/bash_aliases (100%) rename config/{ => shell}/bash_prompt (100%) rename config/{ => shell}/inputrc (100%) rename config/{ => shell}/shell_aliases (100%) rename {systemscripts => config/shell}/solarized.dircolors (100%) rename config/{ => visidata}/access_log.vdj (100%) rename config/{visidataplugins => visidata/plugins}/hidecol.py (100%) rename config/{ => visidata}/visidatarc (100%) create mode 100644 dockerfiles/.DS_Store rename {tools => projects/go-tools}/go/csv2json/csv2json.go (100%) rename {tools => projects/go-tools}/go/gobetween/gobetween.go (100%) rename {tools => projects/go-tools}/go/goinfo/go.mod (100%) rename {tools => projects/go-tools}/go/goinfo/goinfo.go (100%) rename {tools => projects/go-tools}/go/goipgrep/ipgrep.go (100%) rename {tools => projects/go-tools}/go/gosoft/gosoft.go (100%) rename {tools => projects/go-tools}/go/gouniq/gouniq.go (100%) rename {codegrab => projects}/puzzlebox/solution (100%) rename {codegrab => projects}/puzzlebox/solve.py (100%) rename {codegrab => projects}/puzzlebox/solve0.py (100%) rename {codegrab => projects}/puzzlebox/solve2.py (100%) rename {codegrab => projects}/puzzlebox/solve_mp.py (100%) rename {codegrab => projects}/puzzlebox/voxels.py (100%) create mode 100644 projects/rust-tools/between.rs create mode 100644 projects/rust-tools/uniq.rs create mode 100644 projects/rust-tools/uniq2.rs rename {codegrab => projects/timesketch}/deploy_timesketch.sh (100%) create mode 100755 restructure_git.sh create mode 100755 scripts/display/reset_screens.sh rename {systemscripts => scripts/display}/toggle_display.sh (100%) rename {systemscripts => scripts/display}/toggle_touchpad (100%) rename {systemscripts => scripts}/proxy/get_proxy.sh (100%) rename {systemscripts => scripts}/proxy/update_apt_proxy.sh (100%) rename {systemscripts => scripts}/proxy/update_bashrc_proxy.sh (100%) rename {systemscripts => scripts}/proxy/update_service_proxy.sh (100%) rename {systemscripts => scripts/setup}/automountctl (100%) rename {systemscripts => scripts/setup}/share.sh (100%) rename {systemscripts => scripts/setup}/terminal-logs.sh (100%) rename {codegrab => scripts/windows}/Get-ZimmermanTools.ps1 (100%) rename {codegrab/powershell => scripts/windows}/getscreen.psm1 (100%) rename {codegrab => scripts/windows}/sbom.ps1 (100%) create mode 100644 tools/.DS_Store rename {codegrab => tools/cloud}/cloudsend.py (100%) rename {codegrab => tools/cloud}/cloudsend.sh (100%) rename {codegrab => tools/cloud}/speech.py (100%) rename {codegrab => tools/cloud}/vqa3.py (100%) rename {codegrab => tools/cloud}/youtube_resolve.sh (100%) rename {codegrab => tools}/ctf/filtertext.py (100%) rename {codegrab => tools}/ctf/getjs.py (100%) rename {codegrab => tools}/ctf/guess.py (100%) rename {codegrab => tools}/ctf/ps_.py (100%) rename {codegrab => tools}/ctf/search.py (100%) rename {codegrab => tools}/ctf/submit_flag.sh (100%) rename {codegrab => tools}/ctf/transpose.py (100%) rename tools/{ => data}/between (100%) rename tools/{ => data}/concat.py (100%) rename tools/{ => data}/csv_get (100%) rename tools/{ => data}/domgrep.py (100%) rename tools/{ => data}/geturls.py (100%) rename {codegrab => tools/data}/json_save.py (100%) rename {codegrab => tools/data}/kv_parse.py (100%) rename tools/{ => data}/quickchardet.py (100%) rename tools/{ => data}/split_linewise.py (100%) rename {codegrab => tools/data}/uniq.py (100%) rename tools/{ => data}/unum.py (100%) rename tools/{ => data}/urldecode.py (100%) rename {codegrab => tools/data}/vba_chr_decode.py (100%) rename {codegrab => tools/forensics}/chechsqlite.py (100%) rename {codegrab => tools/forensics}/extractfolder.py (100%) rename {codegrab => tools/forensics}/process_leak.py (100%) rename {codegrab => tools/formats}/convert2pdf.sh (100%) rename {codegrab => tools/formats}/flatpdf.sh (100%) rename tools/{ => formats}/rename.mime.py (100%) rename {codegrab => tools/hashing}/hashzip.py (100%) rename tools/{ => hashing}/libarchivesum.py (100%) rename tools/{ => hashing}/scatterhash.py (100%) rename tools/{ => hashing}/tarsum.py (100%) rename {codegrab => tools/network}/fritzshark.sh (100%) rename {codegrab => tools/network}/fritzshark2.sh (100%) rename tools/{ => network}/get_ntp.py (100%) rename tools/{ => network}/get_stp.sh (100%) rename tools/{ => network}/ipgrep (100%) rename {codegrab => tools/security}/certwipe (100%) rename {codegrab => tools/security}/imphash.py (100%) rename {codegrab => tools/security}/scan_vt.py (100%) rename {codegrab => tools/security}/scapy_arp.py (100%) rename {codegrab => tools/security}/simple_portscan.py (100%) rename {codegrab => tools/security}/smtpbanner.py (100%) rename {codegrab => tools/security}/testpw.py (100%) rename {codegrab => tools/security}/vt_download.py (100%) rename {codegrab => tools/security}/vt_ip.py (100%) rename {codegrab => tools/security}/vt_pdns.py (100%) rename tools/{ => system}/backup_docker.sh (100%) rename {codegrab => tools/system}/ltop.py (100%) rename tools/{ => system}/restore_docker.sh (100%) rename tools/{ => system}/watchgrowth.sh (100%) rename {codegrab => tools/system}/wipe.sh (100%) rename {codegrab => tools/text}/depth (100%) rename {codegrab => tools/text}/probability.py (100%) create mode 100755 what diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..39eb21f29d676aedd27cba2bb237e14afb858196 GIT binary patch literal 8196 zcmeHMzfTlF6n-P8IYLM%NsP%hmXcg$OfatGEUivD66JQ`B}al{@q06KFvI>p8zb?}W!|^UdvE6Bo4cERdqkvG_RfGAK_fNS>_Ezy9sXdKkbh@Y|tH z?Aah>c6|T3-{&-W_-d+GFo(~kE+*ii3g$6~z0=^b4TW|M8&+%-G8%lTkMy~H`hK~W zHS_u`Z;lT5+QVX=M)VGycQgkkV7)~{{C$n{-Uc`~MEkt6k2}+SZp^rPH~wBmz~7!~ zcfnNSmCNy;XwMQr)#0tr@g$iI(;j6`M$)}5dp1d`&hu7^>Y3jCB%pV zw?}~)OLdX!{}=Vo|8LJfBuqsCQQ(giP_1sSyMi=VTbCx_+ouTq4LUcDD=eB67;}dO hG&cgk%O4DJ9U+u+VuyuAjG)Pb03(AKQQ)sC@C)&h56%Dp literal 0 HcmV?d00001 diff --git a/.what_db.json b/.what_db.json new file mode 100644 index 0000000..283543f --- /dev/null +++ b/.what_db.json @@ -0,0 +1,149 @@ +{ + "version": "1.0", + "tools": { + "tools/security/scan_vt.py": { + "path": "tools/security/scan_vt.py", + "name": "scan_vt.py", + "type": "python script", + "summary": "Scans files against VirusTotal using MD5 hashes and displays detection results with positives/total ratios and permalink.", + "purpose": "Malware detection and threat analysis", + "short_description": "VirusTotal file scanner with detection ratios", + "executable": true + }, + "tools/security/imphash.py": { + "path": "tools/security/imphash.py", + "name": "imphash.py", + "type": "python script", + "summary": "Calculates and displays the import hash (imphash) of PE files using pefile library for malware analysis.", + "purpose": "Malware analysis and PE file fingerprinting", + "short_description": "PE import hash calculator", + "executable": true + }, + "tools/security/scapy_arp.py": { + "path": "tools/security/scapy_arp.py", + "name": "scapy_arp.py", + "type": "python script", + "summary": "Multi-threaded ARP network scanner using Scapy to discover live hosts on a /24 network range with MAC addresses.", + "purpose": "Network discovery and reconnaissance", + "short_description": "threaded ARP network scanner", + "executable": true + }, + "tools/data/domgrep.py": { + "path": "tools/data/domgrep.py", + "name": "domgrep.py", + "type": "python script", + "summary": "Extracts domain names from URLs read from stdin, filtering out IP addresses and handling malformed URLs gracefully.", + "purpose": "Data extraction and URL processing", + "short_description": "extract domains from URL lists", + "executable": true + }, + "tools/data/unum.py": { + "path": "tools/data/unum.py", + "name": "unum.py", + "type": "python script", + "summary": "Analyzes Unicode characters showing decimal/hex codes, categories, and official Unicode names with proper formatting.", + "purpose": "Text analysis and Unicode debugging", + "short_description": "detailed Unicode character analyzer", + "executable": true + }, + "tools/forensics/chechsqlite.py": { + "path": "tools/forensics/chechsqlite.py", + "name": "chechsqlite.py", + "type": "python script", + "summary": "Scans SQLite databases for tables containing password or hash-related columns for security analysis.", + "purpose": "Database security analysis", + "short_description": "find password/hash columns in SQLite DBs", + "executable": true + }, + "tools/hashing/scatterhash.py": { + "path": "tools/hashing/scatterhash.py", + "name": "scatterhash.py", + "type": "python script", + "summary": "Performs sparse hashing of large files by sampling blocks across the file for efficient integrity checking and validation.", + "purpose": "Large file integrity verification", + "short_description": "sparse hashing for huge files", + "executable": true + }, + "tools/hashing/libarchivesum.py": { + "path": "tools/hashing/libarchivesum.py", + "name": "libarchivesum.py", + "type": "python script", + "summary": "Calculates hashes of individual files within archives (zip, tar, etc.) without extracting them.", + "purpose": "Archive analysis and integrity checking", + "short_description": "like md5sum but for files inside archives", + "executable": true + }, + "tools/system/ltop.py": { + "path": "tools/system/ltop.py", + "name": "ltop.py", + "type": "python script", + "summary": "Real-time frequency counter for stdin lines, showing top N most common entries with live updates using curses.", + "purpose": "Log analysis and monitoring", + "short_description": "like top but for line frequency in streams", + "executable": true + }, + "tools/network/ipgrep": { + "path": "tools/network/ipgrep", + "name": "ipgrep", + "type": "shell script", + "summary": "Comprehensive IP and MAC address extractor with sorting, deduplication, ping testing, and DNS resolution capabilities.", + "purpose": "Network analysis and IP processing", + "short_description": "advanced IP/MAC extractor with ping testing", + "executable": true + }, + "tools/security/certwipe": { + "path": "tools/security/certwipe", + "name": "certwipe", + "type": "shell script", + "summary": "Professional disk wiping tool supporting ATA SecureErase with frozen disk handling and fallback to dc3dd overwriting.", + "purpose": "Data destruction and security", + "short_description": "professional disk wiper with SecureErase", + "executable": true + }, + "tools/system/watchgrowth.sh": { + "path": "tools/system/watchgrowth.sh", + "name": "watchgrowth.sh", + "type": "shell script", + "summary": "Monitors file/directory size growth in real-time, showing transfer speeds and optional progress percentage.", + "purpose": "File monitoring and transfer analysis", + "short_description": "real-time file growth monitor", + "executable": true + }, + "projects/timesketch/deploy_timesketch.sh": { + "path": "projects/timesketch/deploy_timesketch.sh", + "name": "deploy_timesketch.sh", + "type": "shell script", + "summary": "Automated deployment script for Timesketch digital forensics timeline analysis platform with Docker Compose setup.", + "purpose": "Digital forensics infrastructure deployment", + "short_description": "deploy Timesketch forensic timeline platform", + "executable": true + }, + "tools/system/backup_docker.sh": { + "path": "tools/system/backup_docker.sh", + "name": "backup_docker.sh", + "type": "shell script", + "summary": "Comprehensive Docker Compose stack backup including images, configs, and volumes with incremental storage optimization.", + "purpose": "Container infrastructure backup", + "short_description": "backup entire Docker Compose stacks", + "executable": true + }, + "tools/cloud/cloudsend.py": { + "path": "tools/cloud/cloudsend.py", + "name": "cloudsend.py", + "type": "python script", + "summary": "Uploads files to NextCloud/OwnCloud public shares with optional GPG encryption support via command line interface.", + "purpose": "Cloud file sharing and backup", + "short_description": "upload files to NextCloud public shares", + "executable": true + }, + "tools/cloud/vqa3.py": { + "path": "tools/cloud/vqa3.py", + "name": "vqa3.py", + "type": "python script", + "summary": "AI-powered image classification using OpenAI CLIP models for content categorization with customizable classification categories.", + "purpose": "AI image analysis and content filtering", + "short_description": "AI image classifier using CLIP models", + "executable": true + } + } +} diff --git a/WARP.md b/WARP.md new file mode 100644 index 0000000..12f96fe --- /dev/null +++ b/WARP.md @@ -0,0 +1,141 @@ +# WARP.md + +This file provides guidance to WARP (warp.dev) when working with code in this repository. + +## Repository Overview + +This is a collection of utility scripts, tools, and gists organized for cybersecurity, forensics, data analysis, and system administration tasks. The repository contains standalone utilities rather than a cohesive application, with scripts written in Python, Bash, Go, JavaScript, PowerShell, and C. + +## Key Directory Structure + +- **`codegrab/`** - Main collection of security and analysis tools + - `ctf/` - CTF challenge solving scripts + - `puzzlebox/` - 3D puzzle solving algorithms with visualization +- **`tools/`** - System utilities and data processing tools +- **`config/`** - System configuration and installation scripts +- **`systemscripts/`** - System administration and environment setup + - `proxy/` - Network proxy configuration utilities +- **`dockerfiles/`** - Docker container build scripts +- **`collected/`** - Archive of older utilities with documentation + +## Common Development Tasks + +### Running Security Analysis Tools + +Most security tools are standalone and follow this pattern: +```bash +# VirusTotal scanning +./codegrab/scan_vt.py + +# Import hash calculation +python3 codegrab/imphash.py + +# Network analysis +./codegrab/scapy_arp.py +./codegrab/simple_portscan.py +``` + +### Data Processing Utilities + +```bash +# Hash utilities for archives +python3 tools/libarchivesum.py archive.zip + +# Unicode character analysis +echo "text" | python3 tools/unum.py + +# Domain extraction from URLs +cat urls.txt | python3 tools/domgrep.py + +# File organization by MIME type +python3 tools/rename.mime.py +``` + +### Docker Environment Management + +```bash +# Backup Docker Compose stacks +./tools/backup_docker.sh docker-compose.yml + +# Restore Docker environments +./tools/restore_docker.sh + +# Build forensics containers +./dockerfiles/build_kali.sh +``` + +### System Configuration + +```bash +# Install dependencies and configure environment +./config/install.sh + +# Proxy configuration +./systemscripts/proxy/get_proxy.sh +./systemscripts/proxy/update_apt_proxy.sh +``` + +## Architecture and Patterns + +### Security Tools Pattern +Most security utilities in `codegrab/` follow this pattern: +- Standalone executables with shebang +- Take file paths or stdin as input +- Output results in structured format (often CSV-like with custom separators) +- Use external APIs (VirusTotal, etc.) with API keys from `~/.virustotal_api_key` + +### Data Processing Pattern +Tools in `tools/` directory typically: +- Accept multiple file inputs via command line arguments +- Use argparse for option handling +- Support multiple hash algorithms or processing modes +- Include error handling for malformed inputs + +### System Scripts Pattern +Scripts in `systemscripts/` are designed for: +- Environment detection and configuration +- Proxy and network setup automation +- Service management and monitoring +- Display and hardware management + +### Specialized Solvers +The `puzzlebox/` directory contains algorithmic solvers featuring: +- 3D spatial problem solving with numpy +- Visualization using matplotlib +- Recursive backtracking algorithms +- Multi-processing optimization variants + +## Key Dependencies + +The repository relies on various Python packages that should be available: +- **Security**: `pefile`, `requests`, `scapy` +- **Data Processing**: `libarchive-c`, `openpyxl`, `visidata` +- **Scientific**: `numpy`, `matplotlib`, `scipy` +- **Forensics**: `AnalyzeMFT`, `pymisp` +- **System**: `ntplib`, `mac-vendor-lookup`, `dateparser` + +## API Keys and Configuration + +Several tools expect API keys in home directory files: +- `~/.virustotal_api_key` - VirusTotal API access +- Tools may also use environment variables for proxy configuration (`http_proxy`, etc.) + +## Testing and Validation + +Tools are typically tested individually: +```bash +# Test with sample data +python3 codegrab/chechsqlite.py sample.db +python3 tools/quickchardet.py sample.txt + +# Validate with CTF challenges +python3 codegrab/ctf/solve.py +``` + +## Development Notes + +- Most utilities are designed as single-file executables for easy deployment +- Scripts include minimal error handling suitable for command-line usage +- Many tools output to stdout in formats suitable for piping to other commands +- Docker-based tools assume availability of container runtime +- Forensics tools may require elevated privileges for certain operations diff --git a/codegrab/ips.awk b/archive/awk/ips.awk similarity index 100% rename from codegrab/ips.awk rename to archive/awk/ips.awk diff --git a/codegrab/map.awk b/archive/awk/map.awk similarity index 100% rename from codegrab/map.awk rename to archive/awk/map.awk diff --git a/codegrab/csv_cols b/archive/binaries/csv_cols similarity index 100% rename from codegrab/csv_cols rename to archive/binaries/csv_cols diff --git a/codegrab/mapping b/archive/binaries/mapping similarity index 100% rename from codegrab/mapping rename to archive/binaries/mapping diff --git a/collected/README.md b/archive/collected/README.md similarity index 100% rename from collected/README.md rename to archive/collected/README.md diff --git a/collected/appid.tsv b/archive/collected/appid.tsv similarity index 100% rename from collected/appid.tsv rename to archive/collected/appid.tsv diff --git a/collected/commands b/archive/collected/commands similarity index 100% rename from collected/commands rename to archive/collected/commands diff --git a/collected/toollist b/archive/collected/toollist similarity index 100% rename from collected/toollist rename to archive/collected/toollist diff --git a/codegrab/flm.py b/archive/experimental/flm.py similarity index 100% rename from codegrab/flm.py rename to archive/experimental/flm.py diff --git a/codegrab/fuzz.sh b/archive/experimental/fuzz.sh similarity index 100% rename from codegrab/fuzz.sh rename to archive/experimental/fuzz.sh diff --git a/codegrab/hydrogentest.py b/archive/experimental/hydrogentest.py similarity index 100% rename from codegrab/hydrogentest.py rename to archive/experimental/hydrogentest.py diff --git a/codegrab/kv.py b/archive/experimental/kv.py similarity index 100% rename from codegrab/kv.py rename to archive/experimental/kv.py diff --git a/codegrab/lpic.sh b/archive/experimental/lpic.sh similarity index 100% rename from codegrab/lpic.sh rename to archive/experimental/lpic.sh diff --git a/codegrab/matplottest.py b/archive/experimental/matplottest.py similarity index 100% rename from codegrab/matplottest.py rename to archive/experimental/matplottest.py diff --git a/codegrab/.DS_Store b/codegrab/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..586aee470e5230a10355620deb2410250495d506 GIT binary patch literal 6148 zcmeHKK~BRk5L~yFXn{kINO16l2lNM_3KB=8egLHfdPqnFR3J`1ao`<1fJ48)znJw} zsiUMt;(!p^t?Y5uyY^(FW@92U)74@^G$x`U8e?ySuEThqTg!UEa|krN#$+}xi>ei; zj=!q_Kf6B7>5*>o+xYpN?rmm@GEK9*%-|WGTokV#>(8@zcCll&TE?q7q|rWARMHb& z(>*=V5|~ro+}*6V#l>8zx0*}mnnk%`_FZg~C;uO41fDB;!7m5L44j{vEHCf6qhgN7 z?={Dc^mJk z-S8I~khwd;oK-k-%-d@I?r5G*2ct<@R5tMb^CsV-Ow(~*j^Ptsd=!r_uWy%|{j*>B zSGVh9(IFkv14cZDX8P7i7C9|w3QKGB`|@^giCE8}e|E}G=kJok)~Hn`KaXnWBj2Bk zGvEw313O~?HCrS)H1yUPa0Z-#B?Iz*h@pZ>z}irM9cXk50PMo7g1)Y0fUy{WNx<3= z76@A?&_X$EG1$UkkAq(lur{=CVtq316Q0arhr(ec#j&~*Cx+fS1I|E`fjv9zN&Ub2 ze*fPL@-t_^8TeNWa5o*LL!=aHYb!aawK4P>Dk6Th;W7jrEyeJaQhWkcfjyQEFbP;2 R!UFMyfYRWNGq6(zJ^_dxVq*XR literal 0 HcmV?d00001 diff --git a/systemscripts/agnoster.zsh-theme b/config/shell/agnoster.zsh-theme similarity index 100% rename from systemscripts/agnoster.zsh-theme rename to config/shell/agnoster.zsh-theme diff --git a/config/bash_aliases b/config/shell/bash_aliases similarity index 100% rename from config/bash_aliases rename to config/shell/bash_aliases diff --git a/config/bash_prompt b/config/shell/bash_prompt similarity index 100% rename from config/bash_prompt rename to config/shell/bash_prompt diff --git a/config/inputrc b/config/shell/inputrc similarity index 100% rename from config/inputrc rename to config/shell/inputrc diff --git a/config/shell_aliases b/config/shell/shell_aliases similarity index 100% rename from config/shell_aliases rename to config/shell/shell_aliases diff --git a/systemscripts/solarized.dircolors b/config/shell/solarized.dircolors similarity index 100% rename from systemscripts/solarized.dircolors rename to config/shell/solarized.dircolors diff --git a/config/access_log.vdj b/config/visidata/access_log.vdj similarity index 100% rename from config/access_log.vdj rename to config/visidata/access_log.vdj diff --git a/config/visidataplugins/hidecol.py b/config/visidata/plugins/hidecol.py similarity index 100% rename from config/visidataplugins/hidecol.py rename to config/visidata/plugins/hidecol.py diff --git a/config/visidatarc b/config/visidata/visidatarc similarity index 100% rename from config/visidatarc rename to config/visidata/visidatarc diff --git a/dockerfiles/.DS_Store b/dockerfiles/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ac420c372eae7127a3a714792631f2b451c7aad1 GIT binary patch literal 8196 zcmeI1yKWOf6o${lCYW3#8l*Hx3ldNuMBLCIArmfpn$O|(ZuEo>ZbZlen{ z9_OvnHq0;khzj$GGCH9Vji$%#UPT>$hz>*tq65)^=sglBPbW$d>G8KxF?3kB`oK#20*hdGV162pOc5l&u#;Ds*-THly%@#)fC8U{U z-EM!H^?9tB%s+kq@#}e5H@uW?F>d#YoN^a$(GWxfd8cqnkLZk2I^uT=g-!^AI+yS1 zUCURaEf_D-`kaz~6P)bb0-tlAPu+pS8+W;O?pdyqrr@d5_#EO_1Ltr`CqBXE6A+jY zH}|uLHn$xuvQ|g^dvL^T_eHK$Foi zEkTr`ANx++&OubsxJ-`^EmJ^OP_5GWTrZIlKJm3NqB-uel=?o^#;vNtW!imVnM(U{ zeX3XaJmS-iK3}+_QZS`(?Hq1RXz7#f{svjC)@96IS;iIm!oSPteeQksu?23v-j?ul z4nNO)UiBxWaXE*tEoWt4uA8fLK9~4;19$%j*WD>RrvWeX|` z_f{(deAjS4WcW7$6@GrX{Of}kq?jo(vDd!OgYt io::Result<()> { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("Usage: {} start-end [count] [file...]", args[0]); + std::process::exit(1); + } + + let range = &args[1]; + let (start, mut end) = if let Some(dash_pos) = range.find('-') { + (range[..dash_pos].parse().unwrap(), range[dash_pos + 1..].parse().unwrap()) + } else { + (range.parse().unwrap(), 0) + }; + + let mut count = 1; + let mut files_start = 2; + + if end == 0 { + if args.len() > 2 { + if let Ok(c) = args[2].parse::() { + count = c; + files_start = 3; + } + } + end = start + count - 1; + } + + if args.len() > files_start { + for filename in &args[files_start..] { + let file = File::open(filename)?; + let reader = BufReader::new(file); + process_lines(reader, start, end)?; + } + } else { + // No files provided, read from stdin + let stdin = io::stdin(); + let reader = stdin.lock(); + process_lines(reader, start, end)?; + } + + Ok(()) +} + +fn process_lines(reader: R, start: usize, end: usize) -> io::Result<()> { + reader.lines() + .enumerate() + .filter_map(|(i, line)| if i + 1 >= start && i + 1 <= end { line.ok() } else { None }) + .for_each(|line| println!("{}", line)); + + Ok(()) +} diff --git a/projects/rust-tools/uniq.rs b/projects/rust-tools/uniq.rs new file mode 100644 index 0000000..bb82d7e --- /dev/null +++ b/projects/rust-tools/uniq.rs @@ -0,0 +1,41 @@ +use std::env; +use std::fs::File; +use std::io::{self, BufRead, BufReader}; +use std::collections::HashSet; +use std::hash::{Hash, Hasher}; +use std::collections::hash_map::DefaultHasher; + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + + if args.len() > 1 { + for filename in &args[1..] { + let file = File::open(filename)?; + let reader = BufReader::new(file); + remove_duplicates(reader)?; + } + } else { + // No files provided, read from stdin + let stdin = io::stdin(); + let reader = stdin.lock(); + remove_duplicates(reader)?; + } + + Ok(()) +} + +fn remove_duplicates(reader: R) -> io::Result<()> { + let mut seen_hashes = HashSet::new(); + for line in reader.lines() { + let line = line?; + let mut hasher = DefaultHasher::new(); + line.hash(&mut hasher); + let hash = hasher.finish(); + + if seen_hashes.insert(hash) { + println!("{}", line); + } + } + + Ok(()) +} diff --git a/projects/rust-tools/uniq2.rs b/projects/rust-tools/uniq2.rs new file mode 100644 index 0000000..b044bad --- /dev/null +++ b/projects/rust-tools/uniq2.rs @@ -0,0 +1,36 @@ +use std::collections::HashSet; +use std::hash::{Hash, Hasher}; +use std::collections::hash_map::DefaultHasher; + +struct HashOnlySet { + set: HashSet, +} + +impl HashOnlySet { + fn new() -> HashOnlySet { + HashOnlySet { set: HashSet::new() } + } + + fn insert(&mut self, item: &T) -> bool { + let hash = Self::hash_item(item); + self.set.insert(hash) + } + + fn contains(&self, item: &T) -> bool { + let hash = Self::hash_item(item); + self.set.contains(&hash) + } + + fn hash_item(item: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + item.hash(&mut hasher); + hasher.finish() + } +} + +fn main() { + let mut set = HashOnlySet::new(); + set.insert(&"Hello, world!"); + println!("Contains 'Hello, world!': {}", set.contains(&"Hello, world!")); + println!("Contains 'Goodbye, world!': {}", set.contains(&"Goodbye, world!")); +} diff --git a/codegrab/deploy_timesketch.sh b/projects/timesketch/deploy_timesketch.sh similarity index 100% rename from codegrab/deploy_timesketch.sh rename to projects/timesketch/deploy_timesketch.sh diff --git a/restructure_git.sh b/restructure_git.sh new file mode 100755 index 0000000..7359684 --- /dev/null +++ b/restructure_git.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +# Git-aware repository restructuring script +# Uses git mv to preserve file history during reorganization + +set -e + +echo "=== Git-aware Repository Restructuring ===" +echo "This script will reorganize files using 'git mv' to preserve history" +echo "" + +# Check if we're in a git repository +if ! git rev-parse --git-dir > /dev/null 2>&1; then + echo "Warning: Not in a git repository. Using regular 'mv' commands." + MV_CMD="mv" +else + echo "Git repository detected. Using 'git mv' to preserve history." + MV_CMD="git mv" +fi + +echo "" +echo "=== Creating new directory structure ===" + +# Create new directory structure +mkdir -p tools/{security,forensics,data,hashing,network,formats,cloud,system,ctf,text} +mkdir -p projects/{go-tools,puzzlebox,timesketch,rust-tools} +mkdir -p scripts/{proxy,display,setup,windows} +mkdir -p config/{shell,visidata/plugins,applications} +mkdir -p archive/{collected,experimental,binaries,awk} + +echo "=== Moving security tools ===" +$MV_CMD codegrab/scan_vt.py tools/security/ +$MV_CMD codegrab/vt_download.py tools/security/ +$MV_CMD codegrab/vt_ip.py tools/security/ +$MV_CMD codegrab/vt_pdns.py tools/security/ +$MV_CMD codegrab/imphash.py tools/security/ +$MV_CMD codegrab/scapy_arp.py tools/security/ +$MV_CMD codegrab/simple_portscan.py tools/security/ +$MV_CMD codegrab/smtpbanner.py tools/security/ +$MV_CMD codegrab/testpw.py tools/security/ +$MV_CMD codegrab/certwipe tools/security/ + +echo "=== Moving forensics tools ===" +$MV_CMD codegrab/chechsqlite.py tools/forensics/ +$MV_CMD codegrab/process_leak.py tools/forensics/ +$MV_CMD codegrab/extractfolder.py tools/forensics/ + +echo "=== Moving data processing tools ===" +$MV_CMD tools/domgrep.py tools/data/ +$MV_CMD tools/geturls.py tools/data/ +$MV_CMD tools/urldecode.py tools/data/ +$MV_CMD tools/unum.py tools/data/ +$MV_CMD codegrab/vba_chr_decode.py tools/data/ +$MV_CMD tools/quickchardet.py tools/data/ +$MV_CMD codegrab/kv_parse.py tools/data/ +$MV_CMD tools/concat.py tools/data/ +$MV_CMD tools/split_linewise.py tools/data/ +$MV_CMD codegrab/json_save.py tools/data/ +$MV_CMD tools/csv_get tools/data/ +$MV_CMD codegrab/uniq.py tools/data/ +$MV_CMD tools/between tools/data/ + +echo "=== Moving hashing tools ===" +$MV_CMD tools/libarchivesum.py tools/hashing/ +$MV_CMD tools/tarsum.py tools/hashing/ +$MV_CMD codegrab/hashzip.py tools/hashing/ +$MV_CMD tools/scatterhash.py tools/hashing/ + +echo "=== Moving network tools ===" +$MV_CMD tools/ipgrep tools/network/ +$MV_CMD codegrab/fritzshark.sh tools/network/ +$MV_CMD codegrab/fritzshark2.sh tools/network/ +$MV_CMD tools/get_stp.sh tools/network/ +$MV_CMD tools/get_ntp.py tools/network/ + +echo "=== Moving format conversion tools ===" +$MV_CMD codegrab/convert2pdf.sh tools/formats/ +$MV_CMD codegrab/flatpdf.sh tools/formats/ +$MV_CMD tools/rename.mime.py tools/formats/ + +echo "=== Moving cloud service tools ===" +$MV_CMD codegrab/cloudsend.py tools/cloud/ +$MV_CMD codegrab/cloudsend.sh tools/cloud/ +$MV_CMD codegrab/speech.py tools/cloud/ +$MV_CMD codegrab/vqa3.py tools/cloud/ +$MV_CMD codegrab/youtube_resolve.sh tools/cloud/ + +echo "=== Moving system utilities ===" +$MV_CMD tools/backup_docker.sh tools/system/ +$MV_CMD tools/restore_docker.sh tools/system/ +$MV_CMD tools/watchgrowth.sh tools/system/ +$MV_CMD codegrab/wipe.sh tools/system/ +$MV_CMD codegrab/ltop.py tools/system/ + +echo "=== Moving CTF tools ===" +$MV_CMD codegrab/ctf/filtertext.py tools/ctf/ +$MV_CMD codegrab/ctf/getjs.py tools/ctf/ +$MV_CMD codegrab/ctf/guess.py tools/ctf/ +$MV_CMD codegrab/ctf/search.py tools/ctf/ +$MV_CMD codegrab/ctf/transpose.py tools/ctf/ +$MV_CMD codegrab/ctf/ps_.py tools/ctf/ +$MV_CMD codegrab/ctf/submit_flag.sh tools/ctf/ + +echo "=== Moving text analysis tools ===" +$MV_CMD codegrab/probability.py tools/text/ +$MV_CMD codegrab/depth tools/text/ + +echo "=== Moving experimental tools to archive ===" +$MV_CMD codegrab/kv.py archive/experimental/ +$MV_CMD codegrab/flm.py archive/experimental/ +$MV_CMD codegrab/hydrogentest.py archive/experimental/ +$MV_CMD codegrab/matplottest.py archive/experimental/ +$MV_CMD codegrab/lpic.sh archive/experimental/ +$MV_CMD codegrab/fuzz.sh archive/experimental/ + +echo "=== Moving multi-file projects ===" +$MV_CMD tools/go projects/go-tools +$MV_CMD codegrab/puzzlebox projects/ +$MV_CMD codegrab/deploy_timesketch.sh projects/timesketch/ + +# Move Rust tools if they exist +if [ -d tools/rs ]; then + $MV_CMD tools/rs projects/rust-tools +fi + +echo "=== Moving system scripts ===" +$MV_CMD systemscripts/proxy scripts/ +$MV_CMD systemscripts/reset_screens.sh scripts/display/ +$MV_CMD systemscripts/toggle_display.sh scripts/display/ +$MV_CMD systemscripts/toggle_touchpad scripts/display/ +$MV_CMD systemscripts/terminal-logs.sh scripts/setup/ +$MV_CMD systemscripts/automountctl scripts/setup/ + +# Move additional system scripts if they exist +[ -f systemscripts/mount_container ] && $MV_CMD systemscripts/mount_container scripts/setup/ +[ -f systemscripts/fullhd ] && $MV_CMD systemscripts/fullhd scripts/setup/ +[ -f systemscripts/share.sh ] && $MV_CMD systemscripts/share.sh scripts/setup/ + +echo "=== Moving PowerShell scripts ===" +$MV_CMD codegrab/Get-ZimmermanTools.ps1 scripts/windows/ +$MV_CMD codegrab/sbom.ps1 scripts/windows/ +if [ -d codegrab/powershell ]; then + $MV_CMD codegrab/powershell/getscreen.psm1 scripts/windows/ +fi + +echo "=== Organizing configuration files ===" +$MV_CMD config/bash_aliases config/shell/ +$MV_CMD config/bash_prompt config/shell/ +$MV_CMD config/shell_aliases config/shell/ +$MV_CMD config/inputrc config/shell/ +$MV_CMD systemscripts/agnoster.zsh-theme config/shell/ +$MV_CMD systemscripts/solarized.dircolors config/shell/ + +$MV_CMD config/visidatarc config/visidata/ +$MV_CMD config/visidataplugins/hidecol.py config/visidata/plugins/ +[ -f config/access_log.vdj ] && $MV_CMD config/access_log.vdj config/visidata/ + +# Applications directory should already be in the right place +# Just ensure it exists +mkdir -p config/applications + +echo "=== Moving items to archive ===" +$MV_CMD collected archive/ + +# Move binaries and scripts to archive +[ -f codegrab/csv_cols ] && $MV_CMD codegrab/csv_cols archive/binaries/ +[ -f codegrab/mapping ] && $MV_CMD codegrab/mapping archive/binaries/ +[ -f tools/csv2dot ] && $MV_CMD tools/csv2dot archive/binaries/ +[ -f tools/mailunpack ] && $MV_CMD tools/mailunpack archive/binaries/ +[ -f tools/noerr ] && $MV_CMD tools/noerr archive/binaries/ +[ -f tools/openflattenpdf.sh ] && $MV_CMD tools/openflattenpdf.sh archive/binaries/ +[ -f tools/sep_test.sh ] && $MV_CMD tools/sep_test.sh archive/binaries/ +[ -f tools/showgm.sh ] && $MV_CMD tools/showgm.sh archive/binaries/ +[ -f tools/showosm.sh ] && $MV_CMD tools/showosm.sh archive/binaries/ +[ -f tools/sparsecmp.sh ] && $MV_CMD tools/sparsecmp.sh archive/binaries/ +[ -f tools/trunc_by_hash.py ] && $MV_CMD tools/trunc_by_hash.py archive/binaries/ + +# Move AWK scripts +[ -f codegrab/ips.awk ] && $MV_CMD codegrab/ips.awk archive/awk/ +[ -f codegrab/map.awk ] && $MV_CMD codegrab/map.awk archive/awk/ + +# Move any remaining compiled binaries +[ -f codegrab/rootshell.c ] && $MV_CMD codegrab/rootshell.c archive/binaries/ +[ -f codegrab/usbreset.c ] && $MV_CMD codegrab/usbreset.c archive/binaries/ +[ -f codegrab/imphash.go ] && $MV_CMD codegrab/imphash.go archive/binaries/ + +echo "=== Cleaning up empty directories ===" +# Remove empty directories (but be careful with git) +if [ "$MV_CMD" = "git mv" ]; then + echo "Leaving directory cleanup for manual review due to git" +else + rmdir codegrab/ctf codegrab/powershell codegrab systemscripts tools config/visidataplugins 2>/dev/null || true +fi + +echo "" +echo "=== Repository restructuring complete! ===" +echo "" +echo "New structure:" +echo "├── tools/ - Single-file utilities by purpose" +echo "├── projects/ - Multi-file projects" +echo "├── scripts/ - System management scripts" +echo "├── config/ - Configuration files" +echo "├── dockerfiles/ - Docker configurations (unchanged)" +echo "└── archive/ - Legacy and experimental items" +echo "" + +if [ "$MV_CMD" = "git mv" ]; then + echo "All moves used 'git mv' - file history preserved!" + echo "You can review changes with: git status" + echo "Commit when ready with: git commit -m 'Restructure repository for better organization'" +else + echo "Standard 'mv' used - consider initializing git if needed" +fi diff --git a/scripts/display/reset_screens.sh b/scripts/display/reset_screens.sh new file mode 100755 index 0000000..50555f0 --- /dev/null +++ b/scripts/display/reset_screens.sh @@ -0,0 +1,4 @@ +#!/bin/bash +xrandr --output DVI-I-1 --mode 1920x1080 --rotate left --pos 0x0 +xrandr --output DP-1 --primary --mode 2560x1440 --pos 1080x350 +xrandr --output DP-2 --mode 2560x1440 --pos 3640x350 diff --git a/systemscripts/toggle_display.sh b/scripts/display/toggle_display.sh similarity index 100% rename from systemscripts/toggle_display.sh rename to scripts/display/toggle_display.sh diff --git a/systemscripts/toggle_touchpad b/scripts/display/toggle_touchpad similarity index 100% rename from systemscripts/toggle_touchpad rename to scripts/display/toggle_touchpad diff --git a/systemscripts/proxy/get_proxy.sh b/scripts/proxy/get_proxy.sh similarity index 100% rename from systemscripts/proxy/get_proxy.sh rename to scripts/proxy/get_proxy.sh diff --git a/systemscripts/proxy/update_apt_proxy.sh b/scripts/proxy/update_apt_proxy.sh similarity index 100% rename from systemscripts/proxy/update_apt_proxy.sh rename to scripts/proxy/update_apt_proxy.sh diff --git a/systemscripts/proxy/update_bashrc_proxy.sh b/scripts/proxy/update_bashrc_proxy.sh similarity index 100% rename from systemscripts/proxy/update_bashrc_proxy.sh rename to scripts/proxy/update_bashrc_proxy.sh diff --git a/systemscripts/proxy/update_service_proxy.sh b/scripts/proxy/update_service_proxy.sh similarity index 100% rename from systemscripts/proxy/update_service_proxy.sh rename to scripts/proxy/update_service_proxy.sh diff --git a/systemscripts/automountctl b/scripts/setup/automountctl similarity index 100% rename from systemscripts/automountctl rename to scripts/setup/automountctl diff --git a/systemscripts/share.sh b/scripts/setup/share.sh similarity index 100% rename from systemscripts/share.sh rename to scripts/setup/share.sh diff --git a/systemscripts/terminal-logs.sh b/scripts/setup/terminal-logs.sh similarity index 100% rename from systemscripts/terminal-logs.sh rename to scripts/setup/terminal-logs.sh diff --git a/codegrab/Get-ZimmermanTools.ps1 b/scripts/windows/Get-ZimmermanTools.ps1 similarity index 100% rename from codegrab/Get-ZimmermanTools.ps1 rename to scripts/windows/Get-ZimmermanTools.ps1 diff --git a/codegrab/powershell/getscreen.psm1 b/scripts/windows/getscreen.psm1 similarity index 100% rename from codegrab/powershell/getscreen.psm1 rename to scripts/windows/getscreen.psm1 diff --git a/codegrab/sbom.ps1 b/scripts/windows/sbom.ps1 similarity index 100% rename from codegrab/sbom.ps1 rename to scripts/windows/sbom.ps1 diff --git a/tools/.DS_Store b/tools/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ce09f2716542e1320f9e84fa9809ca22bedfcf61 GIT binary patch literal 6148 zcmeHKy-veG47Nj&BC%v-ya#}x6Kkl#6O@qwl}cr3bQOp_6R*LW1@nHatuKhWkAF{=*lcuoGg2s#d?)&d$;fA z`|BXjp3#EhaxtE6Jb$v+v8ravI73{1e6T**Uq2qUIdzsy_7_j?u|GPYI6M%)BBfH6|{vbNyO2bA`b`g6@2l|gdBE% # Find tools matching query + what -h # Show help + what -l # List all tools with short descriptions + what -a # Add new file to database +""" + +import os +import sys +import json +import argparse +import subprocess +import shutil +from pathlib import Path +import re + +# Configuration +REPO_ROOT = Path(__file__).parent.absolute() +DB_FILE = REPO_ROOT / ".what_db.json" + +class WhatTool: + def __init__(self): + self.db_path = DB_FILE + self.data = self.load_db() + + # Detect available tools + self.has_ollama = self.check_ollama() + self.has_fzf = shutil.which('fzf') is not None + + def load_db(self): + """Load the tool database""" + if self.db_path.exists(): + try: + with open(self.db_path, 'r') as f: + return json.load(f) + except json.JSONDecodeError: + print(f"Warning: Corrupted database {self.db_path}, creating new one") + + return { + "version": "1.0", + "tools": {} + } + + def save_db(self): + """Save the tool database""" + with open(self.db_path, 'w') as f: + json.dump(self.data, f, indent=2, sort_keys=True) + + def check_ollama(self): + """Check if ollama with gemma2 is available""" + try: + result = subprocess.run(['ollama', 'list'], capture_output=True, text=True, timeout=5) + if result.returncode == 0: + # Check if gemma2 model is available + models = result.stdout.lower() + return 'gemma2' in models + except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): + pass + return False + + def get_file_type(self, filepath): + """Determine file type""" + if not filepath.exists(): + return "missing" + + if filepath.is_dir(): + return "directory" + + # Check if executable + is_executable = os.access(filepath, os.X_OK) + + # Check extension + suffix = filepath.suffix.lower() + + if suffix == '.py': + return "python script" if is_executable else "python module" + elif suffix == '.sh': + return "shell script" + elif suffix == '.go': + return "go program" + elif suffix == '.js': + return "javascript" + elif suffix == '.ps1': + return "powershell script" + elif suffix == '.rs': + return "rust program" + elif suffix in ['.c', '.cpp']: + return "c/c++ source" + elif suffix == '.awk': + return "awk script" + elif not suffix and is_executable: + return "binary executable" + elif not suffix: + return "script" + else: + return f"{suffix[1:]} file" + + def analyze_file_with_ollama(self, filepath): + """Analyze file using Ollama Gemma2""" + try: + # Read file content (limit size for analysis) + content = "" + if filepath.stat().st_size > 50000: # Skip very large files + content = "[File too large for analysis]" + else: + try: + with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read()[:10000] # First 10KB + except: + content = "[Binary or unreadable file]" + + prompt = f""" +Analyze this code/script file and provide ONLY a JSON response with these fields: + +Filename: {filepath.name} +File type: {self.get_file_type(filepath)} +Content preview: +{content[:2000]} + +Respond with ONLY this JSON structure: +{{ + "summary": "Brief 1-2 sentence summary of what this tool does and how it works", + "purpose": "What this tool is used for (e.g., 'Network analysis', 'File processing', 'Security scanning')", + "short_description": "Very short description for listings (e.g., 'like md5sum but for files inside tarballs')" +}} +""" + + result = subprocess.run([ + 'ollama', 'run', 'gemma2:2b', prompt + ], capture_output=True, text=True, timeout=30) + + if result.returncode == 0: + # Extract JSON from response + response = result.stdout.strip() + + # Try to find JSON in the response + json_match = re.search(r'\{.*\}', response, re.DOTALL) + if json_match: + return json.loads(json_match.group()) + + except (subprocess.TimeoutExpired, json.JSONDecodeError, Exception) as e: + print(f"Ollama analysis failed: {e}") + + return None + + def add_file_interactive(self, filepath): + """Add file with interactive prompts""" + rel_path = str(filepath.relative_to(REPO_ROOT)) + file_type = self.get_file_type(filepath) + + print(f"\nAdding: {rel_path}") + print(f"Type: {file_type}") + print() + + if self.has_ollama: + print("Analyzing with Ollama Gemma2...") + analysis = self.analyze_file_with_ollama(filepath) + + if analysis: + print("AI Analysis complete. Review and edit if needed:") + summary = input(f"Summary [{analysis.get('summary', '')}]: ").strip() + purpose = input(f"Purpose [{analysis.get('purpose', '')}]: ").strip() + short_desc = input(f"Short description [{analysis.get('short_description', '')}]: ").strip() + + # Use AI suggestions if user didn't provide alternatives + summary = summary or analysis.get('summary', '') + purpose = purpose or analysis.get('purpose', '') + short_desc = short_desc or analysis.get('short_description', '') + else: + print("AI analysis failed, using manual input:") + summary = input("Summary (what it does and how): ").strip() + purpose = input("Purpose (what it's used for): ").strip() + short_desc = input("Short description (for listings): ").strip() + else: + print("Manual input (Ollama not available):") + summary = input("Summary (what it does and how): ").strip() + purpose = input("Purpose (what it's used for): ").strip() + short_desc = input("Short description (for listings): ").strip() + + # Store in database + self.data["tools"][rel_path] = { + "path": rel_path, + "name": filepath.name, + "type": file_type, + "summary": summary, + "purpose": purpose, + "short_description": short_desc, + "executable": os.access(filepath, os.X_OK) + } + + self.save_db() + print(f"✓ Added {rel_path} to database") + + def search_with_ollama(self, query): + """Search using natural language with Ollama""" + try: + tools_info = [] + for tool_data in self.data["tools"].values(): + tools_info.append(f"{tool_data['name']}: {tool_data['summary']} (Purpose: {tool_data['purpose']})") + + tools_text = "\n".join(tools_info) + + prompt = f""" +Given this query: "{query}" + +Find the most relevant tools from this list. Respond with ONLY the tool names (one per line) in order of relevance: + +{tools_text} + +Query: {query} + +Response (tool names only, one per line, max 10): +""" + + result = subprocess.run([ + 'ollama', 'run', 'gemma2:2b', prompt + ], capture_output=True, text=True, timeout=20) + + if result.returncode == 0: + tool_names = [line.strip() for line in result.stdout.strip().split('\n') if line.strip()] + + # Find matching tools in database + matches = [] + for tool_name in tool_names[:10]: # Limit to top 10 + for tool_data in self.data["tools"].values(): + if tool_data['name'] == tool_name: + matches.append(tool_data) + break + + return matches + + except Exception as e: + print(f"Ollama search failed: {e}") + + return None + + def search_with_fzf(self, query): + """Search using fzf fuzzy finder""" + try: + # Prepare search data for fzf + search_lines = [] + for tool_data in self.data["tools"].values(): + line = f"{tool_data['name']} # {tool_data['short_description']} | {tool_data['path']}" + search_lines.append(line) + + search_input = "\n".join(search_lines) + + # Run fzf with initial query + result = subprocess.run([ + 'fzf', '--filter', query, '--no-sort' + ], input=search_input, capture_output=True, text=True) + + if result.returncode == 0: + matches = [] + for line in result.stdout.strip().split('\n'): + if ' | ' in line: + path = line.split(' | ')[-1] + if path in self.data["tools"]: + matches.append(self.data["tools"][path]) + + return matches + + except Exception as e: + print(f"fzf search failed: {e}") + + return None + + def search_with_grep(self, query): + """Fallback search using grep-like functionality""" + matches = [] + query_lower = query.lower() + + for tool_data in self.data["tools"].values(): + # Search in name, summary, purpose, and short description + searchable = f"{tool_data['name']} {tool_data['summary']} {tool_data['purpose']} {tool_data['short_description']}".lower() + + if query_lower in searchable: + matches.append(tool_data) + + # Simple relevance scoring + def score_match(tool): + score = 0 + query_lower = query.lower() + if query_lower in tool['name'].lower(): + score += 10 + if query_lower in tool['short_description'].lower(): + score += 5 + if query_lower in tool['summary'].lower(): + score += 3 + if query_lower in tool['purpose'].lower(): + score += 2 + return score + + matches.sort(key=score_match, reverse=True) + return matches[:20] # Limit results + + def search(self, query): + """Search using the best available method""" + if not query: + return [] + + print(f"Searching for: {query}") + + # Try Ollama first + if self.has_ollama: + print("Using Ollama Gemma2 for natural language search...") + results = self.search_with_ollama(query) + if results is not None: + return results + print("Ollama search failed, falling back to fzf...") + + # Try fzf + if self.has_fzf: + print("Using fzf for fuzzy search...") + results = self.search_with_fzf(query) + if results is not None: + return results + print("fzf search failed, falling back to grep...") + + # Fallback to grep + print("Using basic text search...") + return self.search_with_grep(query) + + def list_all_tools(self): + """List all tools with short descriptions""" + if not self.data["tools"]: + print("No tools in database. Use 'what -a ' to add tools.") + return + + print("Available tools:") + print() + + # Sort by name + tools = sorted(self.data["tools"].values(), key=lambda x: x['name']) + + # Calculate max name length for alignment + max_name_len = max(len(tool['name']) for tool in tools) + + for tool in tools: + executable_mark = "*" if tool.get('executable', False) else " " + name_padded = tool['name'].ljust(max_name_len) + print(f"{executable_mark}{name_padded} # {tool['short_description']}") + + def show_search_results(self, results): + """Display search results""" + if not results: + print("No tools found matching your query.") + return + + print(f"\nFound {len(results)} tool(s):") + print() + + for i, tool in enumerate(results, 1): + executable_mark = "*" if tool.get('executable', False) else " " + print(f"{i:2d}. {executable_mark}{tool['name']}") + print(f" Path: {tool['path']}") + print(f" Type: {tool['type']}") + print(f" Purpose: {tool['purpose']}") + print(f" Summary: {tool['summary']}") + print() + +def main(): + parser = argparse.ArgumentParser(description="Smart repository search tool") + parser.add_argument("query", nargs="?", help="Search query") + parser.add_argument("-l", "--list", action="store_true", + help="List all tools with short descriptions") + parser.add_argument("-a", "--add", metavar="PATH", + help="Add new file to database") + + args = parser.parse_args() + + tool = WhatTool() + + if args.list: + tool.list_all_tools() + return + + if args.add: + filepath = Path(args.add) + if not filepath.exists(): + print(f"Error: File {filepath} does not exist") + sys.exit(1) + + if not filepath.is_relative_to(REPO_ROOT): + print(f"Error: File must be within the repository ({REPO_ROOT})") + sys.exit(1) + + tool.add_file_interactive(filepath) + return + + if not args.query: + parser.print_help() + print() + print("Available search methods:") + if tool.has_ollama: + print(" ✓ Ollama + Gemma2 (natural language)") + else: + print(" ✗ Ollama + Gemma2 (not available)") + + if tool.has_fzf: + print(" ✓ fzf (fuzzy finding)") + else: + print(" ✗ fzf (not available)") + + print(" ✓ grep (basic text search)") + return + + # Perform search + results = tool.search(args.query) + tool.show_search_results(results) + +if __name__ == "__main__": + main()