Restructure repository: organize tools by purpose, create what search tool

- Move single-file tools to tools/ organized by category (security, forensics, data, etc.)
- Move multi-file projects to projects/ (go-tools, puzzlebox, timesketch, rust-tools)
- Move system scripts to scripts/ (proxy, display, setup, windows)
- Organize config files in config/ (shell, visidata, applications)
- Move experimental tools to archive/experimental
- Create 'what' fuzzy search tool with progressive enhancement (ollama->fzf->grep)
- Add initial metadata database for intelligent tool discovery
- Preserve git history using 'git mv' commands

tools/hashing/hashzip.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+import zipfile
+import sys
+import hashlib
+
+zip_file_name = sys.argv[1]
+
+with zipfile.ZipFile(zip_file_name, 'r') as zf:
+    print(f"======== Filelisting for {zip_file_name} ========")
+    for file_info in zf.filelist:
+        date_time = file_info.date_time
+        with zf.open(file_info) as zip_file:
+            content = zip_file.read()
+            md5 = hashlib.md5(content).hexdigest()
+        print(f"{file_info.filename} ({file_info.file_size}) {md5} {date_time[0]}/{date_time[1]:02}/{date_time[2]:02} {date_time[3]:02}:{date_time[4]:02}:{date_time[5]:02}")

tools/hashing/libarchivesum.py

@@ -0,0 +1,34 @@
+import hashlib
+import argparse
+import sys
+
+try:
+    import libarchive
+except ImportError:
+    sys.stderr.write('The libarchive package is missing. Please install libarchive-c first.\n\nsudo apt install python3-libarchive-c\n\n')
+    sys.exit(1)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('infile', nargs="+")
+parser.add_argument('-c', '--hashtype', default="md5", choices=hashlib.algorithms_available)
+args = parser.parse_args()
+
+def hash_files_in_archive(archive_path):
+    try:
+        with libarchive.file_reader(archive_path) as archive:
+            for entry in archive:
+                hasher = hashlib.new(args.hashtype)
+                for block in entry.get_blocks():
+                    hasher.update(block)
+                print(f'{hasher.hexdigest()}  {entry.pathname}')
+    except AttributeError:
+        sys.stderr.write("It seems you're using a version of libarchive that doesn't have the 'file_reader' attribute. "
+                         "This script requires 'python-libarchive-c'. Please ensure it's installed.\n")
+        sys.exit(2)
+
+# Example usage:
+for infile in args.infile:
+    try:
+        hash_files_in_archive(infile)
+    except libarchive.exception.ArchiveError as e:
+        print(f'Failed to open archive file {infile}: {e}\n', file=sys.stderr)

tools/hashing/scatterhash.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+import sys
+import hashlib
+import os
+import numpy as np
+import math
+import argparse
+
+def even_select(N, M):
+    if M > N/2:
+        cut = np.zeros(N, dtype=int)
+        q, r = divmod(N, N-M)
+        indices = [q*i + min(i, r) for i in range(N-M)]
+        cut[indices] = True
+    else:
+        cut = np.ones(N, dtype=int)
+        q, r = divmod(N, M)
+        indices = [q*i + min(i, r) for i in range(M)]
+        cut[indices] = False
+    return cut
+
+def get_offsets(blocksize, blockcount,blocks_to_hash):
+    selection = even_select(blockcount,blocks_to_hash)
+    for i in range(0,blockcount):
+        if selection[i] == 0:
+            offset = int(blocksize*i)
+            yield offset
+
+def get_hash(file,hashalgo,spread=-1,maxsize=-1,blocks_to_hash=-1):
+    h = hashlib.new(hashalgo)
+    filesize = os.path.getsize(file.name)
+    fs_block_size = os.stat(file.name).st_blksize
+    if fs_block_size % h.block_size != 0:
+        raise ValueError(f"Filesystem block size {fs_block_size} is not a multiple of hash block size {h.block_size}")
+    blocksize = fs_block_size
+    blockcount = math.ceil(filesize/blocksize)
+    if blocks_to_hash == -1 :
+        blocks_to_hash = math.ceil(blockcount*spread/100)
+        if (blocks_to_hash * blocksize) > maxsize:
+            blocks_to_hash = math.ceil(maxsize/blocksize)
+    if filesize > blocksize:
+        for offset in get_offsets(blocksize, blockcount, blocks_to_hash):
+            file.seek(offset)
+            data = file.read(blocksize)
+            for i in range(0, len(data), h.block_size):
+                h.update(data[i:i + h.block_size])
+    else:
+        h.update(file.read(blocksize))
+    result = f"{h.hexdigest()};{blocks_to_hash};{filesize};{hashalgo};{file.name}"
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Sparsely hash large files. Only a given percentage of the file is actually hashed.')
+    parser.add_argument('-p', metavar='N', dest="spread", type=int, default=10, help='Percentage of file to hash. 0 < N < 100 (default=10)')
+    parser.add_argument('-s', metavar='N', dest="size", type=int, default=10, help='Maximum amount of data per file in MB')
+    parser.add_argument('-c', dest="hashalgo", default="md5", help='Select a hash algorithm (default=md5)')
+    parser.add_argument('file', type=argparse.FileType('rb'), nargs='+')
+    parser.add_argument('-v', dest="validate", action='store_true', help='Read output-file of previous run and validate hashes')
+    parser.add_argument('-1', dest="mismatches", action='store_false', help='Suppress mismatches')
+    parser.add_argument('-0', dest="matches", action='store_false', help='Suppress matches')
+
+    args = parser.parse_args()
+
+    hashalgo = args.hashalgo
+    spread = args.spread
+    maxsize = args.size * 1024 * 1024
+    print(args)
+    if not args.validate:
+        for infile in args.file:
+            print(get_hash(infile, hashalgo, spread, maxsize))
+    else:
+        print("validating")
+        for line in args.file[0]:
+            line = line.decode().strip()
+            hash, blocks_hashed, filesize, hashalgo, file = line.split(';')
+            blocks_hashed = int(blocks_hashed)
+            filesize = int(filesize)
+            if os.path.isfile(file):
+                if os.path.getsize(file) != filesize:
+                    result="BAD_SIZE"
+                else:
+                    rehash = get_hash(open(file,'rb'), hashalgo, blocks_to_hash=blocks_hashed)
+                    if hash == rehash.split(";")[0]:
+                        result = "OK"
+                    else:
+                        result = "BAD_HASH"
+            else:
+                result="FILE_NOT_FOUND"
+            
+            if args.mismatches and result != "OK":
+                print(f"{result};{line}")
+            elif args.matches and result == "OK":
+                print(f"{result};{line}")
+
+if __name__ == "__main__":
+    main()

tools/hashing/tarsum.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python3 -u
+import tarfile
+import hashlib
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('infile', type=argparse.FileType('rb'))
+parser.add_argument('-c','--hashtype', default="md5" , choices=hashlib.algorithms_available )
+args = parser.parse_args()
+
+tf=tarfile.open(fileobj=args.infile)
+
+for file in tf:
+    if file.isfile():
+        h=hashlib.new(args.hashtype)
+        extracted_file=tf.extractfile(file)
+        for chunk in iter(lambda: extracted_file.read(h.block_size),b''):
+            h.update(chunk)
+        print("{1}  {0}".format(file.name,h.hexdigest()))