diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..39eb21f Binary files /dev/null and b/.DS_Store differ diff --git a/.what_db.json b/.what_db.json new file mode 100644 index 0000000..283543f --- /dev/null +++ b/.what_db.json @@ -0,0 +1,149 @@ +{ + "version": "1.0", + "tools": { + "tools/security/scan_vt.py": { + "path": "tools/security/scan_vt.py", + "name": "scan_vt.py", + "type": "python script", + "summary": "Scans files against VirusTotal using MD5 hashes and displays detection results with positives/total ratios and permalink.", + "purpose": "Malware detection and threat analysis", + "short_description": "VirusTotal file scanner with detection ratios", + "executable": true + }, + "tools/security/imphash.py": { + "path": "tools/security/imphash.py", + "name": "imphash.py", + "type": "python script", + "summary": "Calculates and displays the import hash (imphash) of PE files using pefile library for malware analysis.", + "purpose": "Malware analysis and PE file fingerprinting", + "short_description": "PE import hash calculator", + "executable": true + }, + "tools/security/scapy_arp.py": { + "path": "tools/security/scapy_arp.py", + "name": "scapy_arp.py", + "type": "python script", + "summary": "Multi-threaded ARP network scanner using Scapy to discover live hosts on a /24 network range with MAC addresses.", + "purpose": "Network discovery and reconnaissance", + "short_description": "threaded ARP network scanner", + "executable": true + }, + "tools/data/domgrep.py": { + "path": "tools/data/domgrep.py", + "name": "domgrep.py", + "type": "python script", + "summary": "Extracts domain names from URLs read from stdin, filtering out IP addresses and handling malformed URLs gracefully.", + "purpose": "Data extraction and URL processing", + "short_description": "extract domains from URL lists", + "executable": true + }, + "tools/data/unum.py": { + "path": "tools/data/unum.py", + "name": "unum.py", + "type": "python script", + "summary": "Analyzes Unicode characters showing decimal/hex codes, categories, and official Unicode names with proper formatting.", + "purpose": "Text analysis and Unicode debugging", + "short_description": "detailed Unicode character analyzer", + "executable": true + }, + "tools/forensics/chechsqlite.py": { + "path": "tools/forensics/chechsqlite.py", + "name": "chechsqlite.py", + "type": "python script", + "summary": "Scans SQLite databases for tables containing password or hash-related columns for security analysis.", + "purpose": "Database security analysis", + "short_description": "find password/hash columns in SQLite DBs", + "executable": true + }, + "tools/hashing/scatterhash.py": { + "path": "tools/hashing/scatterhash.py", + "name": "scatterhash.py", + "type": "python script", + "summary": "Performs sparse hashing of large files by sampling blocks across the file for efficient integrity checking and validation.", + "purpose": "Large file integrity verification", + "short_description": "sparse hashing for huge files", + "executable": true + }, + "tools/hashing/libarchivesum.py": { + "path": "tools/hashing/libarchivesum.py", + "name": "libarchivesum.py", + "type": "python script", + "summary": "Calculates hashes of individual files within archives (zip, tar, etc.) without extracting them.", + "purpose": "Archive analysis and integrity checking", + "short_description": "like md5sum but for files inside archives", + "executable": true + }, + "tools/system/ltop.py": { + "path": "tools/system/ltop.py", + "name": "ltop.py", + "type": "python script", + "summary": "Real-time frequency counter for stdin lines, showing top N most common entries with live updates using curses.", + "purpose": "Log analysis and monitoring", + "short_description": "like top but for line frequency in streams", + "executable": true + }, + "tools/network/ipgrep": { + "path": "tools/network/ipgrep", + "name": "ipgrep", + "type": "shell script", + "summary": "Comprehensive IP and MAC address extractor with sorting, deduplication, ping testing, and DNS resolution capabilities.", + "purpose": "Network analysis and IP processing", + "short_description": "advanced IP/MAC extractor with ping testing", + "executable": true + }, + "tools/security/certwipe": { + "path": "tools/security/certwipe", + "name": "certwipe", + "type": "shell script", + "summary": "Professional disk wiping tool supporting ATA SecureErase with frozen disk handling and fallback to dc3dd overwriting.", + "purpose": "Data destruction and security", + "short_description": "professional disk wiper with SecureErase", + "executable": true + }, + "tools/system/watchgrowth.sh": { + "path": "tools/system/watchgrowth.sh", + "name": "watchgrowth.sh", + "type": "shell script", + "summary": "Monitors file/directory size growth in real-time, showing transfer speeds and optional progress percentage.", + "purpose": "File monitoring and transfer analysis", + "short_description": "real-time file growth monitor", + "executable": true + }, + "projects/timesketch/deploy_timesketch.sh": { + "path": "projects/timesketch/deploy_timesketch.sh", + "name": "deploy_timesketch.sh", + "type": "shell script", + "summary": "Automated deployment script for Timesketch digital forensics timeline analysis platform with Docker Compose setup.", + "purpose": "Digital forensics infrastructure deployment", + "short_description": "deploy Timesketch forensic timeline platform", + "executable": true + }, + "tools/system/backup_docker.sh": { + "path": "tools/system/backup_docker.sh", + "name": "backup_docker.sh", + "type": "shell script", + "summary": "Comprehensive Docker Compose stack backup including images, configs, and volumes with incremental storage optimization.", + "purpose": "Container infrastructure backup", + "short_description": "backup entire Docker Compose stacks", + "executable": true + }, + "tools/cloud/cloudsend.py": { + "path": "tools/cloud/cloudsend.py", + "name": "cloudsend.py", + "type": "python script", + "summary": "Uploads files to NextCloud/OwnCloud public shares with optional GPG encryption support via command line interface.", + "purpose": "Cloud file sharing and backup", + "short_description": "upload files to NextCloud public shares", + "executable": true + }, + "tools/cloud/vqa3.py": { + "path": "tools/cloud/vqa3.py", + "name": "vqa3.py", + "type": "python script", + "summary": "AI-powered image classification using OpenAI CLIP models for content categorization with customizable classification categories.", + "purpose": "AI image analysis and content filtering", + "short_description": "AI image classifier using CLIP models", + "executable": true + } + } +} diff --git a/WARP.md b/WARP.md new file mode 100644 index 0000000..12f96fe --- /dev/null +++ b/WARP.md @@ -0,0 +1,141 @@ +# WARP.md + +This file provides guidance to WARP (warp.dev) when working with code in this repository. + +## Repository Overview + +This is a collection of utility scripts, tools, and gists organized for cybersecurity, forensics, data analysis, and system administration tasks. The repository contains standalone utilities rather than a cohesive application, with scripts written in Python, Bash, Go, JavaScript, PowerShell, and C. + +## Key Directory Structure + +- **`codegrab/`** - Main collection of security and analysis tools + - `ctf/` - CTF challenge solving scripts + - `puzzlebox/` - 3D puzzle solving algorithms with visualization +- **`tools/`** - System utilities and data processing tools +- **`config/`** - System configuration and installation scripts +- **`systemscripts/`** - System administration and environment setup + - `proxy/` - Network proxy configuration utilities +- **`dockerfiles/`** - Docker container build scripts +- **`collected/`** - Archive of older utilities with documentation + +## Common Development Tasks + +### Running Security Analysis Tools + +Most security tools are standalone and follow this pattern: +```bash +# VirusTotal scanning +./codegrab/scan_vt.py + +# Import hash calculation +python3 codegrab/imphash.py + +# Network analysis +./codegrab/scapy_arp.py +./codegrab/simple_portscan.py +``` + +### Data Processing Utilities + +```bash +# Hash utilities for archives +python3 tools/libarchivesum.py archive.zip + +# Unicode character analysis +echo "text" | python3 tools/unum.py + +# Domain extraction from URLs +cat urls.txt | python3 tools/domgrep.py + +# File organization by MIME type +python3 tools/rename.mime.py +``` + +### Docker Environment Management + +```bash +# Backup Docker Compose stacks +./tools/backup_docker.sh docker-compose.yml + +# Restore Docker environments +./tools/restore_docker.sh + +# Build forensics containers +./dockerfiles/build_kali.sh +``` + +### System Configuration + +```bash +# Install dependencies and configure environment +./config/install.sh + +# Proxy configuration +./systemscripts/proxy/get_proxy.sh +./systemscripts/proxy/update_apt_proxy.sh +``` + +## Architecture and Patterns + +### Security Tools Pattern +Most security utilities in `codegrab/` follow this pattern: +- Standalone executables with shebang +- Take file paths or stdin as input +- Output results in structured format (often CSV-like with custom separators) +- Use external APIs (VirusTotal, etc.) with API keys from `~/.virustotal_api_key` + +### Data Processing Pattern +Tools in `tools/` directory typically: +- Accept multiple file inputs via command line arguments +- Use argparse for option handling +- Support multiple hash algorithms or processing modes +- Include error handling for malformed inputs + +### System Scripts Pattern +Scripts in `systemscripts/` are designed for: +- Environment detection and configuration +- Proxy and network setup automation +- Service management and monitoring +- Display and hardware management + +### Specialized Solvers +The `puzzlebox/` directory contains algorithmic solvers featuring: +- 3D spatial problem solving with numpy +- Visualization using matplotlib +- Recursive backtracking algorithms +- Multi-processing optimization variants + +## Key Dependencies + +The repository relies on various Python packages that should be available: +- **Security**: `pefile`, `requests`, `scapy` +- **Data Processing**: `libarchive-c`, `openpyxl`, `visidata` +- **Scientific**: `numpy`, `matplotlib`, `scipy` +- **Forensics**: `AnalyzeMFT`, `pymisp` +- **System**: `ntplib`, `mac-vendor-lookup`, `dateparser` + +## API Keys and Configuration + +Several tools expect API keys in home directory files: +- `~/.virustotal_api_key` - VirusTotal API access +- Tools may also use environment variables for proxy configuration (`http_proxy`, etc.) + +## Testing and Validation + +Tools are typically tested individually: +```bash +# Test with sample data +python3 codegrab/chechsqlite.py sample.db +python3 tools/quickchardet.py sample.txt + +# Validate with CTF challenges +python3 codegrab/ctf/solve.py +``` + +## Development Notes + +- Most utilities are designed as single-file executables for easy deployment +- Scripts include minimal error handling suitable for command-line usage +- Many tools output to stdout in formats suitable for piping to other commands +- Docker-based tools assume availability of container runtime +- Forensics tools may require elevated privileges for certain operations diff --git a/codegrab/ips.awk b/archive/awk/ips.awk similarity index 100% rename from codegrab/ips.awk rename to archive/awk/ips.awk diff --git a/codegrab/map.awk b/archive/awk/map.awk similarity index 100% rename from codegrab/map.awk rename to archive/awk/map.awk diff --git a/codegrab/csv_cols b/archive/binaries/csv_cols similarity index 100% rename from codegrab/csv_cols rename to archive/binaries/csv_cols diff --git a/codegrab/mapping b/archive/binaries/mapping similarity index 100% rename from codegrab/mapping rename to archive/binaries/mapping diff --git a/collected/README.md b/archive/collected/README.md similarity index 100% rename from collected/README.md rename to archive/collected/README.md diff --git a/collected/appid.tsv b/archive/collected/appid.tsv similarity index 100% rename from collected/appid.tsv rename to archive/collected/appid.tsv diff --git a/collected/commands b/archive/collected/commands similarity index 100% rename from collected/commands rename to archive/collected/commands diff --git a/collected/toollist b/archive/collected/toollist similarity index 100% rename from collected/toollist rename to archive/collected/toollist diff --git a/codegrab/flm.py b/archive/experimental/flm.py similarity index 100% rename from codegrab/flm.py rename to archive/experimental/flm.py diff --git a/codegrab/fuzz.sh b/archive/experimental/fuzz.sh similarity index 100% rename from codegrab/fuzz.sh rename to archive/experimental/fuzz.sh diff --git a/codegrab/hydrogentest.py b/archive/experimental/hydrogentest.py similarity index 100% rename from codegrab/hydrogentest.py rename to archive/experimental/hydrogentest.py diff --git a/codegrab/kv.py b/archive/experimental/kv.py similarity index 100% rename from codegrab/kv.py rename to archive/experimental/kv.py diff --git a/codegrab/lpic.sh b/archive/experimental/lpic.sh similarity index 100% rename from codegrab/lpic.sh rename to archive/experimental/lpic.sh diff --git a/codegrab/matplottest.py b/archive/experimental/matplottest.py similarity index 100% rename from codegrab/matplottest.py rename to archive/experimental/matplottest.py diff --git a/codegrab/.DS_Store b/codegrab/.DS_Store new file mode 100644 index 0000000..586aee4 Binary files /dev/null and b/codegrab/.DS_Store differ diff --git a/codegrab/imphash.go b/codegrab/imphash.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/codegrab/imphash.go @@ -0,0 +1 @@ +package main diff --git a/config/.DS_Store b/config/.DS_Store new file mode 100644 index 0000000..cf913e2 Binary files /dev/null and b/config/.DS_Store differ diff --git a/systemscripts/agnoster.zsh-theme b/config/shell/agnoster.zsh-theme similarity index 100% rename from systemscripts/agnoster.zsh-theme rename to config/shell/agnoster.zsh-theme diff --git a/config/bash_aliases b/config/shell/bash_aliases similarity index 100% rename from config/bash_aliases rename to config/shell/bash_aliases diff --git a/config/bash_prompt b/config/shell/bash_prompt similarity index 100% rename from config/bash_prompt rename to config/shell/bash_prompt diff --git a/config/inputrc b/config/shell/inputrc similarity index 100% rename from config/inputrc rename to config/shell/inputrc diff --git a/config/shell_aliases b/config/shell/shell_aliases similarity index 100% rename from config/shell_aliases rename to config/shell/shell_aliases diff --git a/systemscripts/solarized.dircolors b/config/shell/solarized.dircolors similarity index 100% rename from systemscripts/solarized.dircolors rename to config/shell/solarized.dircolors diff --git a/config/access_log.vdj b/config/visidata/access_log.vdj similarity index 100% rename from config/access_log.vdj rename to config/visidata/access_log.vdj diff --git a/config/visidataplugins/hidecol.py b/config/visidata/plugins/hidecol.py similarity index 100% rename from config/visidataplugins/hidecol.py rename to config/visidata/plugins/hidecol.py diff --git a/config/visidatarc b/config/visidata/visidatarc similarity index 100% rename from config/visidatarc rename to config/visidata/visidatarc diff --git a/dockerfiles/.DS_Store b/dockerfiles/.DS_Store new file mode 100644 index 0000000..ac420c3 Binary files /dev/null and b/dockerfiles/.DS_Store differ diff --git a/tools/go/csv2json/csv2json.go b/projects/go-tools/go/csv2json/csv2json.go similarity index 100% rename from tools/go/csv2json/csv2json.go rename to projects/go-tools/go/csv2json/csv2json.go diff --git a/tools/go/gobetween/gobetween.go b/projects/go-tools/go/gobetween/gobetween.go similarity index 100% rename from tools/go/gobetween/gobetween.go rename to projects/go-tools/go/gobetween/gobetween.go diff --git a/tools/go/goinfo/go.mod b/projects/go-tools/go/goinfo/go.mod similarity index 100% rename from tools/go/goinfo/go.mod rename to projects/go-tools/go/goinfo/go.mod diff --git a/tools/go/goinfo/goinfo.go b/projects/go-tools/go/goinfo/goinfo.go similarity index 100% rename from tools/go/goinfo/goinfo.go rename to projects/go-tools/go/goinfo/goinfo.go diff --git a/tools/go/goipgrep/ipgrep b/projects/go-tools/go/goipgrep/ipgrep similarity index 100% rename from tools/go/goipgrep/ipgrep rename to projects/go-tools/go/goipgrep/ipgrep diff --git a/tools/go/goipgrep/ipgrep.go b/projects/go-tools/go/goipgrep/ipgrep.go similarity index 100% rename from tools/go/goipgrep/ipgrep.go rename to projects/go-tools/go/goipgrep/ipgrep.go diff --git a/tools/go/gosoft/gosoft.exe b/projects/go-tools/go/gosoft/gosoft.exe similarity index 100% rename from tools/go/gosoft/gosoft.exe rename to projects/go-tools/go/gosoft/gosoft.exe diff --git a/tools/go/gosoft/gosoft.go b/projects/go-tools/go/gosoft/gosoft.go similarity index 100% rename from tools/go/gosoft/gosoft.go rename to projects/go-tools/go/gosoft/gosoft.go diff --git a/tools/go/gouniq/gouniq.go b/projects/go-tools/go/gouniq/gouniq.go similarity index 100% rename from tools/go/gouniq/gouniq.go rename to projects/go-tools/go/gouniq/gouniq.go diff --git a/codegrab/puzzlebox/solution b/projects/puzzlebox/solution similarity index 100% rename from codegrab/puzzlebox/solution rename to projects/puzzlebox/solution diff --git a/codegrab/puzzlebox/solve.py b/projects/puzzlebox/solve.py similarity index 100% rename from codegrab/puzzlebox/solve.py rename to projects/puzzlebox/solve.py diff --git a/codegrab/puzzlebox/solve0.py b/projects/puzzlebox/solve0.py similarity index 100% rename from codegrab/puzzlebox/solve0.py rename to projects/puzzlebox/solve0.py diff --git a/codegrab/puzzlebox/solve2.py b/projects/puzzlebox/solve2.py similarity index 100% rename from codegrab/puzzlebox/solve2.py rename to projects/puzzlebox/solve2.py diff --git a/codegrab/puzzlebox/solve_mp.py b/projects/puzzlebox/solve_mp.py similarity index 100% rename from codegrab/puzzlebox/solve_mp.py rename to projects/puzzlebox/solve_mp.py diff --git a/codegrab/puzzlebox/voxels.py b/projects/puzzlebox/voxels.py similarity index 100% rename from codegrab/puzzlebox/voxels.py rename to projects/puzzlebox/voxels.py diff --git a/projects/rust-tools/between.rs b/projects/rust-tools/between.rs new file mode 100644 index 0000000..c7aa9cd --- /dev/null +++ b/projects/rust-tools/between.rs @@ -0,0 +1,55 @@ +use std::env; +use std::fs::File; +use std::io::{self, BufRead, BufReader}; + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("Usage: {} start-end [count] [file...]", args[0]); + std::process::exit(1); + } + + let range = &args[1]; + let (start, mut end) = if let Some(dash_pos) = range.find('-') { + (range[..dash_pos].parse().unwrap(), range[dash_pos + 1..].parse().unwrap()) + } else { + (range.parse().unwrap(), 0) + }; + + let mut count = 1; + let mut files_start = 2; + + if end == 0 { + if args.len() > 2 { + if let Ok(c) = args[2].parse::() { + count = c; + files_start = 3; + } + } + end = start + count - 1; + } + + if args.len() > files_start { + for filename in &args[files_start..] { + let file = File::open(filename)?; + let reader = BufReader::new(file); + process_lines(reader, start, end)?; + } + } else { + // No files provided, read from stdin + let stdin = io::stdin(); + let reader = stdin.lock(); + process_lines(reader, start, end)?; + } + + Ok(()) +} + +fn process_lines(reader: R, start: usize, end: usize) -> io::Result<()> { + reader.lines() + .enumerate() + .filter_map(|(i, line)| if i + 1 >= start && i + 1 <= end { line.ok() } else { None }) + .for_each(|line| println!("{}", line)); + + Ok(()) +} diff --git a/projects/rust-tools/uniq b/projects/rust-tools/uniq new file mode 100755 index 0000000..64da4a9 Binary files /dev/null and b/projects/rust-tools/uniq differ diff --git a/projects/rust-tools/uniq.rs b/projects/rust-tools/uniq.rs new file mode 100644 index 0000000..bb82d7e --- /dev/null +++ b/projects/rust-tools/uniq.rs @@ -0,0 +1,41 @@ +use std::env; +use std::fs::File; +use std::io::{self, BufRead, BufReader}; +use std::collections::HashSet; +use std::hash::{Hash, Hasher}; +use std::collections::hash_map::DefaultHasher; + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + + if args.len() > 1 { + for filename in &args[1..] { + let file = File::open(filename)?; + let reader = BufReader::new(file); + remove_duplicates(reader)?; + } + } else { + // No files provided, read from stdin + let stdin = io::stdin(); + let reader = stdin.lock(); + remove_duplicates(reader)?; + } + + Ok(()) +} + +fn remove_duplicates(reader: R) -> io::Result<()> { + let mut seen_hashes = HashSet::new(); + for line in reader.lines() { + let line = line?; + let mut hasher = DefaultHasher::new(); + line.hash(&mut hasher); + let hash = hasher.finish(); + + if seen_hashes.insert(hash) { + println!("{}", line); + } + } + + Ok(()) +} diff --git a/projects/rust-tools/uniq2 b/projects/rust-tools/uniq2 new file mode 100755 index 0000000..5f5ba7d Binary files /dev/null and b/projects/rust-tools/uniq2 differ diff --git a/projects/rust-tools/uniq2.rs b/projects/rust-tools/uniq2.rs new file mode 100644 index 0000000..b044bad --- /dev/null +++ b/projects/rust-tools/uniq2.rs @@ -0,0 +1,36 @@ +use std::collections::HashSet; +use std::hash::{Hash, Hasher}; +use std::collections::hash_map::DefaultHasher; + +struct HashOnlySet { + set: HashSet, +} + +impl HashOnlySet { + fn new() -> HashOnlySet { + HashOnlySet { set: HashSet::new() } + } + + fn insert(&mut self, item: &T) -> bool { + let hash = Self::hash_item(item); + self.set.insert(hash) + } + + fn contains(&self, item: &T) -> bool { + let hash = Self::hash_item(item); + self.set.contains(&hash) + } + + fn hash_item(item: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + item.hash(&mut hasher); + hasher.finish() + } +} + +fn main() { + let mut set = HashOnlySet::new(); + set.insert(&"Hello, world!"); + println!("Contains 'Hello, world!': {}", set.contains(&"Hello, world!")); + println!("Contains 'Goodbye, world!': {}", set.contains(&"Goodbye, world!")); +} diff --git a/codegrab/deploy_timesketch.sh b/projects/timesketch/deploy_timesketch.sh similarity index 100% rename from codegrab/deploy_timesketch.sh rename to projects/timesketch/deploy_timesketch.sh diff --git a/restructure_git.sh b/restructure_git.sh new file mode 100755 index 0000000..7359684 --- /dev/null +++ b/restructure_git.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +# Git-aware repository restructuring script +# Uses git mv to preserve file history during reorganization + +set -e + +echo "=== Git-aware Repository Restructuring ===" +echo "This script will reorganize files using 'git mv' to preserve history" +echo "" + +# Check if we're in a git repository +if ! git rev-parse --git-dir > /dev/null 2>&1; then + echo "Warning: Not in a git repository. Using regular 'mv' commands." + MV_CMD="mv" +else + echo "Git repository detected. Using 'git mv' to preserve history." + MV_CMD="git mv" +fi + +echo "" +echo "=== Creating new directory structure ===" + +# Create new directory structure +mkdir -p tools/{security,forensics,data,hashing,network,formats,cloud,system,ctf,text} +mkdir -p projects/{go-tools,puzzlebox,timesketch,rust-tools} +mkdir -p scripts/{proxy,display,setup,windows} +mkdir -p config/{shell,visidata/plugins,applications} +mkdir -p archive/{collected,experimental,binaries,awk} + +echo "=== Moving security tools ===" +$MV_CMD codegrab/scan_vt.py tools/security/ +$MV_CMD codegrab/vt_download.py tools/security/ +$MV_CMD codegrab/vt_ip.py tools/security/ +$MV_CMD codegrab/vt_pdns.py tools/security/ +$MV_CMD codegrab/imphash.py tools/security/ +$MV_CMD codegrab/scapy_arp.py tools/security/ +$MV_CMD codegrab/simple_portscan.py tools/security/ +$MV_CMD codegrab/smtpbanner.py tools/security/ +$MV_CMD codegrab/testpw.py tools/security/ +$MV_CMD codegrab/certwipe tools/security/ + +echo "=== Moving forensics tools ===" +$MV_CMD codegrab/chechsqlite.py tools/forensics/ +$MV_CMD codegrab/process_leak.py tools/forensics/ +$MV_CMD codegrab/extractfolder.py tools/forensics/ + +echo "=== Moving data processing tools ===" +$MV_CMD tools/domgrep.py tools/data/ +$MV_CMD tools/geturls.py tools/data/ +$MV_CMD tools/urldecode.py tools/data/ +$MV_CMD tools/unum.py tools/data/ +$MV_CMD codegrab/vba_chr_decode.py tools/data/ +$MV_CMD tools/quickchardet.py tools/data/ +$MV_CMD codegrab/kv_parse.py tools/data/ +$MV_CMD tools/concat.py tools/data/ +$MV_CMD tools/split_linewise.py tools/data/ +$MV_CMD codegrab/json_save.py tools/data/ +$MV_CMD tools/csv_get tools/data/ +$MV_CMD codegrab/uniq.py tools/data/ +$MV_CMD tools/between tools/data/ + +echo "=== Moving hashing tools ===" +$MV_CMD tools/libarchivesum.py tools/hashing/ +$MV_CMD tools/tarsum.py tools/hashing/ +$MV_CMD codegrab/hashzip.py tools/hashing/ +$MV_CMD tools/scatterhash.py tools/hashing/ + +echo "=== Moving network tools ===" +$MV_CMD tools/ipgrep tools/network/ +$MV_CMD codegrab/fritzshark.sh tools/network/ +$MV_CMD codegrab/fritzshark2.sh tools/network/ +$MV_CMD tools/get_stp.sh tools/network/ +$MV_CMD tools/get_ntp.py tools/network/ + +echo "=== Moving format conversion tools ===" +$MV_CMD codegrab/convert2pdf.sh tools/formats/ +$MV_CMD codegrab/flatpdf.sh tools/formats/ +$MV_CMD tools/rename.mime.py tools/formats/ + +echo "=== Moving cloud service tools ===" +$MV_CMD codegrab/cloudsend.py tools/cloud/ +$MV_CMD codegrab/cloudsend.sh tools/cloud/ +$MV_CMD codegrab/speech.py tools/cloud/ +$MV_CMD codegrab/vqa3.py tools/cloud/ +$MV_CMD codegrab/youtube_resolve.sh tools/cloud/ + +echo "=== Moving system utilities ===" +$MV_CMD tools/backup_docker.sh tools/system/ +$MV_CMD tools/restore_docker.sh tools/system/ +$MV_CMD tools/watchgrowth.sh tools/system/ +$MV_CMD codegrab/wipe.sh tools/system/ +$MV_CMD codegrab/ltop.py tools/system/ + +echo "=== Moving CTF tools ===" +$MV_CMD codegrab/ctf/filtertext.py tools/ctf/ +$MV_CMD codegrab/ctf/getjs.py tools/ctf/ +$MV_CMD codegrab/ctf/guess.py tools/ctf/ +$MV_CMD codegrab/ctf/search.py tools/ctf/ +$MV_CMD codegrab/ctf/transpose.py tools/ctf/ +$MV_CMD codegrab/ctf/ps_.py tools/ctf/ +$MV_CMD codegrab/ctf/submit_flag.sh tools/ctf/ + +echo "=== Moving text analysis tools ===" +$MV_CMD codegrab/probability.py tools/text/ +$MV_CMD codegrab/depth tools/text/ + +echo "=== Moving experimental tools to archive ===" +$MV_CMD codegrab/kv.py archive/experimental/ +$MV_CMD codegrab/flm.py archive/experimental/ +$MV_CMD codegrab/hydrogentest.py archive/experimental/ +$MV_CMD codegrab/matplottest.py archive/experimental/ +$MV_CMD codegrab/lpic.sh archive/experimental/ +$MV_CMD codegrab/fuzz.sh archive/experimental/ + +echo "=== Moving multi-file projects ===" +$MV_CMD tools/go projects/go-tools +$MV_CMD codegrab/puzzlebox projects/ +$MV_CMD codegrab/deploy_timesketch.sh projects/timesketch/ + +# Move Rust tools if they exist +if [ -d tools/rs ]; then + $MV_CMD tools/rs projects/rust-tools +fi + +echo "=== Moving system scripts ===" +$MV_CMD systemscripts/proxy scripts/ +$MV_CMD systemscripts/reset_screens.sh scripts/display/ +$MV_CMD systemscripts/toggle_display.sh scripts/display/ +$MV_CMD systemscripts/toggle_touchpad scripts/display/ +$MV_CMD systemscripts/terminal-logs.sh scripts/setup/ +$MV_CMD systemscripts/automountctl scripts/setup/ + +# Move additional system scripts if they exist +[ -f systemscripts/mount_container ] && $MV_CMD systemscripts/mount_container scripts/setup/ +[ -f systemscripts/fullhd ] && $MV_CMD systemscripts/fullhd scripts/setup/ +[ -f systemscripts/share.sh ] && $MV_CMD systemscripts/share.sh scripts/setup/ + +echo "=== Moving PowerShell scripts ===" +$MV_CMD codegrab/Get-ZimmermanTools.ps1 scripts/windows/ +$MV_CMD codegrab/sbom.ps1 scripts/windows/ +if [ -d codegrab/powershell ]; then + $MV_CMD codegrab/powershell/getscreen.psm1 scripts/windows/ +fi + +echo "=== Organizing configuration files ===" +$MV_CMD config/bash_aliases config/shell/ +$MV_CMD config/bash_prompt config/shell/ +$MV_CMD config/shell_aliases config/shell/ +$MV_CMD config/inputrc config/shell/ +$MV_CMD systemscripts/agnoster.zsh-theme config/shell/ +$MV_CMD systemscripts/solarized.dircolors config/shell/ + +$MV_CMD config/visidatarc config/visidata/ +$MV_CMD config/visidataplugins/hidecol.py config/visidata/plugins/ +[ -f config/access_log.vdj ] && $MV_CMD config/access_log.vdj config/visidata/ + +# Applications directory should already be in the right place +# Just ensure it exists +mkdir -p config/applications + +echo "=== Moving items to archive ===" +$MV_CMD collected archive/ + +# Move binaries and scripts to archive +[ -f codegrab/csv_cols ] && $MV_CMD codegrab/csv_cols archive/binaries/ +[ -f codegrab/mapping ] && $MV_CMD codegrab/mapping archive/binaries/ +[ -f tools/csv2dot ] && $MV_CMD tools/csv2dot archive/binaries/ +[ -f tools/mailunpack ] && $MV_CMD tools/mailunpack archive/binaries/ +[ -f tools/noerr ] && $MV_CMD tools/noerr archive/binaries/ +[ -f tools/openflattenpdf.sh ] && $MV_CMD tools/openflattenpdf.sh archive/binaries/ +[ -f tools/sep_test.sh ] && $MV_CMD tools/sep_test.sh archive/binaries/ +[ -f tools/showgm.sh ] && $MV_CMD tools/showgm.sh archive/binaries/ +[ -f tools/showosm.sh ] && $MV_CMD tools/showosm.sh archive/binaries/ +[ -f tools/sparsecmp.sh ] && $MV_CMD tools/sparsecmp.sh archive/binaries/ +[ -f tools/trunc_by_hash.py ] && $MV_CMD tools/trunc_by_hash.py archive/binaries/ + +# Move AWK scripts +[ -f codegrab/ips.awk ] && $MV_CMD codegrab/ips.awk archive/awk/ +[ -f codegrab/map.awk ] && $MV_CMD codegrab/map.awk archive/awk/ + +# Move any remaining compiled binaries +[ -f codegrab/rootshell.c ] && $MV_CMD codegrab/rootshell.c archive/binaries/ +[ -f codegrab/usbreset.c ] && $MV_CMD codegrab/usbreset.c archive/binaries/ +[ -f codegrab/imphash.go ] && $MV_CMD codegrab/imphash.go archive/binaries/ + +echo "=== Cleaning up empty directories ===" +# Remove empty directories (but be careful with git) +if [ "$MV_CMD" = "git mv" ]; then + echo "Leaving directory cleanup for manual review due to git" +else + rmdir codegrab/ctf codegrab/powershell codegrab systemscripts tools config/visidataplugins 2>/dev/null || true +fi + +echo "" +echo "=== Repository restructuring complete! ===" +echo "" +echo "New structure:" +echo "├── tools/ - Single-file utilities by purpose" +echo "├── projects/ - Multi-file projects" +echo "├── scripts/ - System management scripts" +echo "├── config/ - Configuration files" +echo "├── dockerfiles/ - Docker configurations (unchanged)" +echo "└── archive/ - Legacy and experimental items" +echo "" + +if [ "$MV_CMD" = "git mv" ]; then + echo "All moves used 'git mv' - file history preserved!" + echo "You can review changes with: git status" + echo "Commit when ready with: git commit -m 'Restructure repository for better organization'" +else + echo "Standard 'mv' used - consider initializing git if needed" +fi diff --git a/scripts/display/reset_screens.sh b/scripts/display/reset_screens.sh new file mode 100755 index 0000000..50555f0 --- /dev/null +++ b/scripts/display/reset_screens.sh @@ -0,0 +1,4 @@ +#!/bin/bash +xrandr --output DVI-I-1 --mode 1920x1080 --rotate left --pos 0x0 +xrandr --output DP-1 --primary --mode 2560x1440 --pos 1080x350 +xrandr --output DP-2 --mode 2560x1440 --pos 3640x350 diff --git a/systemscripts/toggle_display.sh b/scripts/display/toggle_display.sh similarity index 100% rename from systemscripts/toggle_display.sh rename to scripts/display/toggle_display.sh diff --git a/systemscripts/toggle_touchpad b/scripts/display/toggle_touchpad similarity index 100% rename from systemscripts/toggle_touchpad rename to scripts/display/toggle_touchpad diff --git a/systemscripts/proxy/get_proxy.sh b/scripts/proxy/get_proxy.sh similarity index 100% rename from systemscripts/proxy/get_proxy.sh rename to scripts/proxy/get_proxy.sh diff --git a/systemscripts/proxy/update_apt_proxy.sh b/scripts/proxy/update_apt_proxy.sh similarity index 100% rename from systemscripts/proxy/update_apt_proxy.sh rename to scripts/proxy/update_apt_proxy.sh diff --git a/systemscripts/proxy/update_bashrc_proxy.sh b/scripts/proxy/update_bashrc_proxy.sh similarity index 100% rename from systemscripts/proxy/update_bashrc_proxy.sh rename to scripts/proxy/update_bashrc_proxy.sh diff --git a/systemscripts/proxy/update_service_proxy.sh b/scripts/proxy/update_service_proxy.sh similarity index 100% rename from systemscripts/proxy/update_service_proxy.sh rename to scripts/proxy/update_service_proxy.sh diff --git a/systemscripts/automountctl b/scripts/setup/automountctl similarity index 100% rename from systemscripts/automountctl rename to scripts/setup/automountctl diff --git a/systemscripts/share.sh b/scripts/setup/share.sh similarity index 100% rename from systemscripts/share.sh rename to scripts/setup/share.sh diff --git a/systemscripts/terminal-logs.sh b/scripts/setup/terminal-logs.sh similarity index 100% rename from systemscripts/terminal-logs.sh rename to scripts/setup/terminal-logs.sh diff --git a/codegrab/Get-ZimmermanTools.ps1 b/scripts/windows/Get-ZimmermanTools.ps1 similarity index 100% rename from codegrab/Get-ZimmermanTools.ps1 rename to scripts/windows/Get-ZimmermanTools.ps1 diff --git a/codegrab/powershell/getscreen.psm1 b/scripts/windows/getscreen.psm1 similarity index 100% rename from codegrab/powershell/getscreen.psm1 rename to scripts/windows/getscreen.psm1 diff --git a/codegrab/sbom.ps1 b/scripts/windows/sbom.ps1 similarity index 100% rename from codegrab/sbom.ps1 rename to scripts/windows/sbom.ps1 diff --git a/tools/.DS_Store b/tools/.DS_Store new file mode 100644 index 0000000..ce09f27 Binary files /dev/null and b/tools/.DS_Store differ diff --git a/codegrab/cloudsend.py b/tools/cloud/cloudsend.py similarity index 100% rename from codegrab/cloudsend.py rename to tools/cloud/cloudsend.py diff --git a/codegrab/cloudsend.sh b/tools/cloud/cloudsend.sh similarity index 100% rename from codegrab/cloudsend.sh rename to tools/cloud/cloudsend.sh diff --git a/codegrab/speech.py b/tools/cloud/speech.py similarity index 100% rename from codegrab/speech.py rename to tools/cloud/speech.py diff --git a/codegrab/vqa3.py b/tools/cloud/vqa3.py similarity index 100% rename from codegrab/vqa3.py rename to tools/cloud/vqa3.py diff --git a/codegrab/youtube_resolve.sh b/tools/cloud/youtube_resolve.sh similarity index 100% rename from codegrab/youtube_resolve.sh rename to tools/cloud/youtube_resolve.sh diff --git a/codegrab/ctf/filtertext.py b/tools/ctf/filtertext.py similarity index 100% rename from codegrab/ctf/filtertext.py rename to tools/ctf/filtertext.py diff --git a/codegrab/ctf/getjs.py b/tools/ctf/getjs.py similarity index 100% rename from codegrab/ctf/getjs.py rename to tools/ctf/getjs.py diff --git a/codegrab/ctf/guess.py b/tools/ctf/guess.py similarity index 100% rename from codegrab/ctf/guess.py rename to tools/ctf/guess.py diff --git a/codegrab/ctf/ps_.py b/tools/ctf/ps_.py similarity index 100% rename from codegrab/ctf/ps_.py rename to tools/ctf/ps_.py diff --git a/codegrab/ctf/search.py b/tools/ctf/search.py similarity index 100% rename from codegrab/ctf/search.py rename to tools/ctf/search.py diff --git a/codegrab/ctf/submit_flag.sh b/tools/ctf/submit_flag.sh similarity index 100% rename from codegrab/ctf/submit_flag.sh rename to tools/ctf/submit_flag.sh diff --git a/codegrab/ctf/transpose.py b/tools/ctf/transpose.py similarity index 100% rename from codegrab/ctf/transpose.py rename to tools/ctf/transpose.py diff --git a/tools/between b/tools/data/between similarity index 100% rename from tools/between rename to tools/data/between diff --git a/tools/concat.py b/tools/data/concat.py similarity index 100% rename from tools/concat.py rename to tools/data/concat.py diff --git a/tools/csv_get b/tools/data/csv_get similarity index 100% rename from tools/csv_get rename to tools/data/csv_get diff --git a/tools/domgrep.py b/tools/data/domgrep.py similarity index 100% rename from tools/domgrep.py rename to tools/data/domgrep.py diff --git a/tools/geturls.py b/tools/data/geturls.py similarity index 100% rename from tools/geturls.py rename to tools/data/geturls.py diff --git a/codegrab/json_save.py b/tools/data/json_save.py similarity index 100% rename from codegrab/json_save.py rename to tools/data/json_save.py diff --git a/codegrab/kv_parse.py b/tools/data/kv_parse.py similarity index 100% rename from codegrab/kv_parse.py rename to tools/data/kv_parse.py diff --git a/tools/quickchardet.py b/tools/data/quickchardet.py similarity index 100% rename from tools/quickchardet.py rename to tools/data/quickchardet.py diff --git a/tools/split_linewise.py b/tools/data/split_linewise.py similarity index 100% rename from tools/split_linewise.py rename to tools/data/split_linewise.py diff --git a/codegrab/uniq.py b/tools/data/uniq.py similarity index 100% rename from codegrab/uniq.py rename to tools/data/uniq.py diff --git a/tools/unum.py b/tools/data/unum.py similarity index 100% rename from tools/unum.py rename to tools/data/unum.py diff --git a/tools/urldecode.py b/tools/data/urldecode.py similarity index 100% rename from tools/urldecode.py rename to tools/data/urldecode.py diff --git a/codegrab/vba_chr_decode.py b/tools/data/vba_chr_decode.py similarity index 100% rename from codegrab/vba_chr_decode.py rename to tools/data/vba_chr_decode.py diff --git a/codegrab/chechsqlite.py b/tools/forensics/chechsqlite.py similarity index 100% rename from codegrab/chechsqlite.py rename to tools/forensics/chechsqlite.py diff --git a/codegrab/extractfolder.py b/tools/forensics/extractfolder.py similarity index 100% rename from codegrab/extractfolder.py rename to tools/forensics/extractfolder.py diff --git a/codegrab/process_leak.py b/tools/forensics/process_leak.py similarity index 100% rename from codegrab/process_leak.py rename to tools/forensics/process_leak.py diff --git a/codegrab/convert2pdf.sh b/tools/formats/convert2pdf.sh similarity index 100% rename from codegrab/convert2pdf.sh rename to tools/formats/convert2pdf.sh diff --git a/codegrab/flatpdf.sh b/tools/formats/flatpdf.sh similarity index 100% rename from codegrab/flatpdf.sh rename to tools/formats/flatpdf.sh diff --git a/tools/rename.mime.py b/tools/formats/rename.mime.py similarity index 100% rename from tools/rename.mime.py rename to tools/formats/rename.mime.py diff --git a/codegrab/hashzip.py b/tools/hashing/hashzip.py similarity index 100% rename from codegrab/hashzip.py rename to tools/hashing/hashzip.py diff --git a/tools/libarchivesum.py b/tools/hashing/libarchivesum.py similarity index 100% rename from tools/libarchivesum.py rename to tools/hashing/libarchivesum.py diff --git a/tools/scatterhash.py b/tools/hashing/scatterhash.py similarity index 100% rename from tools/scatterhash.py rename to tools/hashing/scatterhash.py diff --git a/tools/tarsum.py b/tools/hashing/tarsum.py similarity index 100% rename from tools/tarsum.py rename to tools/hashing/tarsum.py diff --git a/codegrab/fritzshark.sh b/tools/network/fritzshark.sh similarity index 100% rename from codegrab/fritzshark.sh rename to tools/network/fritzshark.sh diff --git a/codegrab/fritzshark2.sh b/tools/network/fritzshark2.sh similarity index 100% rename from codegrab/fritzshark2.sh rename to tools/network/fritzshark2.sh diff --git a/tools/get_ntp.py b/tools/network/get_ntp.py similarity index 100% rename from tools/get_ntp.py rename to tools/network/get_ntp.py diff --git a/tools/get_stp.sh b/tools/network/get_stp.sh similarity index 100% rename from tools/get_stp.sh rename to tools/network/get_stp.sh diff --git a/tools/ipgrep b/tools/network/ipgrep similarity index 100% rename from tools/ipgrep rename to tools/network/ipgrep diff --git a/codegrab/certwipe b/tools/security/certwipe similarity index 100% rename from codegrab/certwipe rename to tools/security/certwipe diff --git a/codegrab/imphash.py b/tools/security/imphash.py similarity index 100% rename from codegrab/imphash.py rename to tools/security/imphash.py diff --git a/codegrab/scan_vt.py b/tools/security/scan_vt.py similarity index 100% rename from codegrab/scan_vt.py rename to tools/security/scan_vt.py diff --git a/codegrab/scapy_arp.py b/tools/security/scapy_arp.py similarity index 100% rename from codegrab/scapy_arp.py rename to tools/security/scapy_arp.py diff --git a/codegrab/simple_portscan.py b/tools/security/simple_portscan.py similarity index 100% rename from codegrab/simple_portscan.py rename to tools/security/simple_portscan.py diff --git a/codegrab/smtpbanner.py b/tools/security/smtpbanner.py similarity index 100% rename from codegrab/smtpbanner.py rename to tools/security/smtpbanner.py diff --git a/codegrab/testpw.py b/tools/security/testpw.py similarity index 100% rename from codegrab/testpw.py rename to tools/security/testpw.py diff --git a/codegrab/vt_download.py b/tools/security/vt_download.py similarity index 100% rename from codegrab/vt_download.py rename to tools/security/vt_download.py diff --git a/codegrab/vt_ip.py b/tools/security/vt_ip.py similarity index 100% rename from codegrab/vt_ip.py rename to tools/security/vt_ip.py diff --git a/codegrab/vt_pdns.py b/tools/security/vt_pdns.py similarity index 100% rename from codegrab/vt_pdns.py rename to tools/security/vt_pdns.py diff --git a/tools/backup_docker.sh b/tools/system/backup_docker.sh similarity index 100% rename from tools/backup_docker.sh rename to tools/system/backup_docker.sh diff --git a/codegrab/ltop.py b/tools/system/ltop.py similarity index 100% rename from codegrab/ltop.py rename to tools/system/ltop.py diff --git a/tools/restore_docker.sh b/tools/system/restore_docker.sh similarity index 100% rename from tools/restore_docker.sh rename to tools/system/restore_docker.sh diff --git a/tools/watchgrowth.sh b/tools/system/watchgrowth.sh similarity index 100% rename from tools/watchgrowth.sh rename to tools/system/watchgrowth.sh diff --git a/codegrab/wipe.sh b/tools/system/wipe.sh similarity index 100% rename from codegrab/wipe.sh rename to tools/system/wipe.sh diff --git a/codegrab/depth b/tools/text/depth similarity index 100% rename from codegrab/depth rename to tools/text/depth diff --git a/codegrab/probability.py b/tools/text/probability.py similarity index 100% rename from codegrab/probability.py rename to tools/text/probability.py diff --git a/what b/what new file mode 100755 index 0000000..24569ad --- /dev/null +++ b/what @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 + +""" +'what' - Smart repository search tool with progressive enhancement + +Fallback hierarchy: +1. Ollama + Gemma2 (natural language search) +2. fzf (fuzzy finding) +3. grep (simple text search) + +Usage: + what # Find tools matching query + what -h # Show help + what -l # List all tools with short descriptions + what -a # Add new file to database +""" + +import os +import sys +import json +import argparse +import subprocess +import shutil +from pathlib import Path +import re + +# Configuration +REPO_ROOT = Path(__file__).parent.absolute() +DB_FILE = REPO_ROOT / ".what_db.json" + +class WhatTool: + def __init__(self): + self.db_path = DB_FILE + self.data = self.load_db() + + # Detect available tools + self.has_ollama = self.check_ollama() + self.has_fzf = shutil.which('fzf') is not None + + def load_db(self): + """Load the tool database""" + if self.db_path.exists(): + try: + with open(self.db_path, 'r') as f: + return json.load(f) + except json.JSONDecodeError: + print(f"Warning: Corrupted database {self.db_path}, creating new one") + + return { + "version": "1.0", + "tools": {} + } + + def save_db(self): + """Save the tool database""" + with open(self.db_path, 'w') as f: + json.dump(self.data, f, indent=2, sort_keys=True) + + def check_ollama(self): + """Check if ollama with gemma2 is available""" + try: + result = subprocess.run(['ollama', 'list'], capture_output=True, text=True, timeout=5) + if result.returncode == 0: + # Check if gemma2 model is available + models = result.stdout.lower() + return 'gemma2' in models + except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): + pass + return False + + def get_file_type(self, filepath): + """Determine file type""" + if not filepath.exists(): + return "missing" + + if filepath.is_dir(): + return "directory" + + # Check if executable + is_executable = os.access(filepath, os.X_OK) + + # Check extension + suffix = filepath.suffix.lower() + + if suffix == '.py': + return "python script" if is_executable else "python module" + elif suffix == '.sh': + return "shell script" + elif suffix == '.go': + return "go program" + elif suffix == '.js': + return "javascript" + elif suffix == '.ps1': + return "powershell script" + elif suffix == '.rs': + return "rust program" + elif suffix in ['.c', '.cpp']: + return "c/c++ source" + elif suffix == '.awk': + return "awk script" + elif not suffix and is_executable: + return "binary executable" + elif not suffix: + return "script" + else: + return f"{suffix[1:]} file" + + def analyze_file_with_ollama(self, filepath): + """Analyze file using Ollama Gemma2""" + try: + # Read file content (limit size for analysis) + content = "" + if filepath.stat().st_size > 50000: # Skip very large files + content = "[File too large for analysis]" + else: + try: + with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read()[:10000] # First 10KB + except: + content = "[Binary or unreadable file]" + + prompt = f""" +Analyze this code/script file and provide ONLY a JSON response with these fields: + +Filename: {filepath.name} +File type: {self.get_file_type(filepath)} +Content preview: +{content[:2000]} + +Respond with ONLY this JSON structure: +{{ + "summary": "Brief 1-2 sentence summary of what this tool does and how it works", + "purpose": "What this tool is used for (e.g., 'Network analysis', 'File processing', 'Security scanning')", + "short_description": "Very short description for listings (e.g., 'like md5sum but for files inside tarballs')" +}} +""" + + result = subprocess.run([ + 'ollama', 'run', 'gemma2:2b', prompt + ], capture_output=True, text=True, timeout=30) + + if result.returncode == 0: + # Extract JSON from response + response = result.stdout.strip() + + # Try to find JSON in the response + json_match = re.search(r'\{.*\}', response, re.DOTALL) + if json_match: + return json.loads(json_match.group()) + + except (subprocess.TimeoutExpired, json.JSONDecodeError, Exception) as e: + print(f"Ollama analysis failed: {e}") + + return None + + def add_file_interactive(self, filepath): + """Add file with interactive prompts""" + rel_path = str(filepath.relative_to(REPO_ROOT)) + file_type = self.get_file_type(filepath) + + print(f"\nAdding: {rel_path}") + print(f"Type: {file_type}") + print() + + if self.has_ollama: + print("Analyzing with Ollama Gemma2...") + analysis = self.analyze_file_with_ollama(filepath) + + if analysis: + print("AI Analysis complete. Review and edit if needed:") + summary = input(f"Summary [{analysis.get('summary', '')}]: ").strip() + purpose = input(f"Purpose [{analysis.get('purpose', '')}]: ").strip() + short_desc = input(f"Short description [{analysis.get('short_description', '')}]: ").strip() + + # Use AI suggestions if user didn't provide alternatives + summary = summary or analysis.get('summary', '') + purpose = purpose or analysis.get('purpose', '') + short_desc = short_desc or analysis.get('short_description', '') + else: + print("AI analysis failed, using manual input:") + summary = input("Summary (what it does and how): ").strip() + purpose = input("Purpose (what it's used for): ").strip() + short_desc = input("Short description (for listings): ").strip() + else: + print("Manual input (Ollama not available):") + summary = input("Summary (what it does and how): ").strip() + purpose = input("Purpose (what it's used for): ").strip() + short_desc = input("Short description (for listings): ").strip() + + # Store in database + self.data["tools"][rel_path] = { + "path": rel_path, + "name": filepath.name, + "type": file_type, + "summary": summary, + "purpose": purpose, + "short_description": short_desc, + "executable": os.access(filepath, os.X_OK) + } + + self.save_db() + print(f"✓ Added {rel_path} to database") + + def search_with_ollama(self, query): + """Search using natural language with Ollama""" + try: + tools_info = [] + for tool_data in self.data["tools"].values(): + tools_info.append(f"{tool_data['name']}: {tool_data['summary']} (Purpose: {tool_data['purpose']})") + + tools_text = "\n".join(tools_info) + + prompt = f""" +Given this query: "{query}" + +Find the most relevant tools from this list. Respond with ONLY the tool names (one per line) in order of relevance: + +{tools_text} + +Query: {query} + +Response (tool names only, one per line, max 10): +""" + + result = subprocess.run([ + 'ollama', 'run', 'gemma2:2b', prompt + ], capture_output=True, text=True, timeout=20) + + if result.returncode == 0: + tool_names = [line.strip() for line in result.stdout.strip().split('\n') if line.strip()] + + # Find matching tools in database + matches = [] + for tool_name in tool_names[:10]: # Limit to top 10 + for tool_data in self.data["tools"].values(): + if tool_data['name'] == tool_name: + matches.append(tool_data) + break + + return matches + + except Exception as e: + print(f"Ollama search failed: {e}") + + return None + + def search_with_fzf(self, query): + """Search using fzf fuzzy finder""" + try: + # Prepare search data for fzf + search_lines = [] + for tool_data in self.data["tools"].values(): + line = f"{tool_data['name']} # {tool_data['short_description']} | {tool_data['path']}" + search_lines.append(line) + + search_input = "\n".join(search_lines) + + # Run fzf with initial query + result = subprocess.run([ + 'fzf', '--filter', query, '--no-sort' + ], input=search_input, capture_output=True, text=True) + + if result.returncode == 0: + matches = [] + for line in result.stdout.strip().split('\n'): + if ' | ' in line: + path = line.split(' | ')[-1] + if path in self.data["tools"]: + matches.append(self.data["tools"][path]) + + return matches + + except Exception as e: + print(f"fzf search failed: {e}") + + return None + + def search_with_grep(self, query): + """Fallback search using grep-like functionality""" + matches = [] + query_lower = query.lower() + + for tool_data in self.data["tools"].values(): + # Search in name, summary, purpose, and short description + searchable = f"{tool_data['name']} {tool_data['summary']} {tool_data['purpose']} {tool_data['short_description']}".lower() + + if query_lower in searchable: + matches.append(tool_data) + + # Simple relevance scoring + def score_match(tool): + score = 0 + query_lower = query.lower() + if query_lower in tool['name'].lower(): + score += 10 + if query_lower in tool['short_description'].lower(): + score += 5 + if query_lower in tool['summary'].lower(): + score += 3 + if query_lower in tool['purpose'].lower(): + score += 2 + return score + + matches.sort(key=score_match, reverse=True) + return matches[:20] # Limit results + + def search(self, query): + """Search using the best available method""" + if not query: + return [] + + print(f"Searching for: {query}") + + # Try Ollama first + if self.has_ollama: + print("Using Ollama Gemma2 for natural language search...") + results = self.search_with_ollama(query) + if results is not None: + return results + print("Ollama search failed, falling back to fzf...") + + # Try fzf + if self.has_fzf: + print("Using fzf for fuzzy search...") + results = self.search_with_fzf(query) + if results is not None: + return results + print("fzf search failed, falling back to grep...") + + # Fallback to grep + print("Using basic text search...") + return self.search_with_grep(query) + + def list_all_tools(self): + """List all tools with short descriptions""" + if not self.data["tools"]: + print("No tools in database. Use 'what -a ' to add tools.") + return + + print("Available tools:") + print() + + # Sort by name + tools = sorted(self.data["tools"].values(), key=lambda x: x['name']) + + # Calculate max name length for alignment + max_name_len = max(len(tool['name']) for tool in tools) + + for tool in tools: + executable_mark = "*" if tool.get('executable', False) else " " + name_padded = tool['name'].ljust(max_name_len) + print(f"{executable_mark}{name_padded} # {tool['short_description']}") + + def show_search_results(self, results): + """Display search results""" + if not results: + print("No tools found matching your query.") + return + + print(f"\nFound {len(results)} tool(s):") + print() + + for i, tool in enumerate(results, 1): + executable_mark = "*" if tool.get('executable', False) else " " + print(f"{i:2d}. {executable_mark}{tool['name']}") + print(f" Path: {tool['path']}") + print(f" Type: {tool['type']}") + print(f" Purpose: {tool['purpose']}") + print(f" Summary: {tool['summary']}") + print() + +def main(): + parser = argparse.ArgumentParser(description="Smart repository search tool") + parser.add_argument("query", nargs="?", help="Search query") + parser.add_argument("-l", "--list", action="store_true", + help="List all tools with short descriptions") + parser.add_argument("-a", "--add", metavar="PATH", + help="Add new file to database") + + args = parser.parse_args() + + tool = WhatTool() + + if args.list: + tool.list_all_tools() + return + + if args.add: + filepath = Path(args.add) + if not filepath.exists(): + print(f"Error: File {filepath} does not exist") + sys.exit(1) + + if not filepath.is_relative_to(REPO_ROOT): + print(f"Error: File must be within the repository ({REPO_ROOT})") + sys.exit(1) + + tool.add_file_interactive(filepath) + return + + if not args.query: + parser.print_help() + print() + print("Available search methods:") + if tool.has_ollama: + print(" ✓ Ollama + Gemma2 (natural language)") + else: + print(" ✗ Ollama + Gemma2 (not available)") + + if tool.has_fzf: + print(" ✓ fzf (fuzzy finding)") + else: + print(" ✗ fzf (not available)") + + print(" ✓ grep (basic text search)") + return + + # Perform search + results = tool.search(args.query) + tool.show_search_results(results) + +if __name__ == "__main__": + main()