Major repository cleanup and enhancement
- Reorganize documentation: moved old docs to docs/ directory - Add comprehensive README.md with build options and usage guide - Add detailed CONTRIBUTING.md with help content management guide - Create Makefile for automated building and testing - Add Dockerfile.scratch for building from Ubuntu 20.04 base - Enhance all Dockerfiles with PowerShell + PSScriptAnalyzer - Add modern shells: zsh (with plugins) and fish (with config) - Add modern CLI tools: fd-find, ripgrep, fzf - Create comprehensive help system with cheat/TLDR/fish completions - Add helper scripts for help content management and coverage checking - Fix Dockerfile.remnux script references - Support three build variants: upstream (REMnux), scratch (Ubuntu), kali Build options: - make build-upstream: Fast, uses REMnux upstream (recommended) - make build-scratch: Full control, builds from Ubuntu 20.04 - make build-kali: Legacy Kali Linux base Features: - PowerShell with PSScriptAnalyzer module - Modern shells (zsh, fish) with custom configurations - Enhanced help system (cheat sheets, TLDR pages, fish completions) - Help coverage checking and bulk import tools - Comprehensive documentation for users and contributors
This commit is contained in:
Tobias Kessels
329
scripts/check-help-coverage.sh
Normal file
329
scripts/check-help-coverage.sh
Normal file
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# check-help-coverage.sh
|
||||
# Check cheat and tldr help coverage for a benchmark list of analysis tools.
|
||||
# Designed to run inside the container (tabledevil/file-analysis:*).
|
||||
# - Cheat sheets are expected under /opt/cheatsheets/personal or /opt/cheatsheets/*.cheat
|
||||
# - TLDR pages are checked under /home/remnux/.local/share/tldr/pages/{common,linux}
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - ran successfully
|
||||
# 1 - minor warnings
|
||||
# 2 - failed to run (e.g., missing directories)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CHEAT_DIR_PERSONAL="/opt/cheatsheets/personal"
|
||||
CHEAT_DIR_GLOBAL="/opt/cheatsheets"
|
||||
TLDR_DIRS=("/home/remnux/.local/share/tldr/pages/common" "/home/remnux/.local/share/tldr/pages/linux")
|
||||
|
||||
# Resolve a cheat file from a name
|
||||
resolve_cheat_file() {
|
||||
local name="$1"
|
||||
local base
|
||||
base="$(echo "$name" | sed 's/\.[Pp][Yy]$//')"
|
||||
|
||||
# candidate names to try in order
|
||||
local candidates=(
|
||||
"$name"
|
||||
"$base"
|
||||
"${base}.py"
|
||||
"${base// /-}"
|
||||
"${base// /_}"
|
||||
"${base//-/_}"
|
||||
"${base//_/-}"
|
||||
)
|
||||
|
||||
for cand in "${candidates[@]}"; do
|
||||
# personal dir (preferred)
|
||||
if [[ -f "$CHEAT_DIR_PERSONAL/$cand" ]]; then
|
||||
echo "$CHEAT_DIR_PERSONAL/$cand"; return 0
|
||||
fi
|
||||
# global .cheat files
|
||||
if [[ -f "$CHEAT_DIR_GLOBAL/${cand}.cheat" ]]; then
|
||||
echo "$CHEAT_DIR_GLOBAL/${cand}.cheat"; return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Check if any TLDR page exists for a given tool name (try variants)
|
||||
resolve_tldr_page() {
|
||||
local name="$1"
|
||||
local base lower
|
||||
base="$(echo "$name" | sed 's/\.[Pp][Yy]$//; s/[() ]//g')"
|
||||
lower="$(echo "$name" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
local candidates=(
|
||||
"$name"
|
||||
"$lower"
|
||||
"$base"
|
||||
"${base}.py"
|
||||
"${lower// /-}"
|
||||
"${lower// /_}"
|
||||
"${lower//-/_}"
|
||||
"${lower//_/-}"
|
||||
)
|
||||
|
||||
for dir in "${TLDR_DIRS[@]}"; do
|
||||
for cand in "${candidates[@]}"; do
|
||||
if [[ -f "$dir/${cand}.md" ]]; then
|
||||
echo "$dir/${cand}.md"; return 0
|
||||
fi
|
||||
done
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Print a line with fixed columns
|
||||
print_row() {
|
||||
printf "%-18s | %-35s | %-6s | %-6s\n" "$1" "$2" "$3" "$4"
|
||||
}
|
||||
|
||||
# Benchmark list: Category | Display Name | Cheat candidates (comma) | TLDR candidates (comma)
|
||||
# Keep names short for display; put multiple candidate names to test resolution logic.
|
||||
TOOL_LIST=$(cat << 'EOF'
|
||||
General | TrID | trid | trid
|
||||
General | Magika | magika | magika
|
||||
General | file | file | file
|
||||
General | Detect-It-Easy | die,diec | die
|
||||
General | ExifTool | exiftool | exiftool
|
||||
General | YARA | yara,yarac | yara
|
||||
General | signsrch | signsrch | signsrch
|
||||
General | ssdeep | ssdeep | ssdeep
|
||||
General | ClamAV | clamscan,freshclam | clamscan,freshclam
|
||||
General | bulk_extractor | bulk_extractor | bulk_extractor
|
||||
General | Hachoir | hachoir-metadata,hachoir-subfile | hachoir-metadata
|
||||
General | Sleuth Kit | fls,icat,tsk_recover | fls,icat
|
||||
General | binwalk | binwalk | binwalk
|
||||
General | 7-Zip | 7z,7za,7zr,7zz | 7z
|
||||
General | wxHexEditor | wxhexeditor | wxhexeditor
|
||||
General | zipdump | zipdump,zipdump.py | zipdump
|
||||
General | msitools | msiinfo,msiextract | msiinfo,msiextract
|
||||
General | numbers-to-string | numbers-to-string,numbers-to-string.py | numbers-to-string
|
||||
General | re-search | re-search,re-search.py | re-search
|
||||
General | disitool | disitool,disitool.py | disitool
|
||||
General | Name-That-Hash | nth | nth
|
||||
General | Hash ID | hashid,hash-id.py | hashid
|
||||
|
||||
PE | manalyze | manalyze | manalyze
|
||||
PE | peframe | peframe | peframe
|
||||
PE | pefile (lib) | pefile | pefile
|
||||
PE | pescan | pescan,pescan.py | pescan
|
||||
PE | portex | portex,portex.py | portex
|
||||
PE | bearcommander | bearcommander | bearcommander
|
||||
PE | pecheck | pecheck | pecheck
|
||||
|
||||
ELF | pyew | pyew | pyew
|
||||
ELF | readelf.py | readelf,readelf.py | readelf
|
||||
|
||||
Strings | pestr | pestr | pestr
|
||||
Strings | bbcrack | bbcrack | bbcrack
|
||||
Strings | brxor | brxor,brxor.py | brxor
|
||||
Strings | base64dump | base64dump,base64dump.py | base64dump
|
||||
Strings | xorsearch | xorsearch | xorsearch
|
||||
Strings | XORStrings | xorstrings | xorstrings
|
||||
Strings | flarestrings | flarestrings | flarestrings
|
||||
Strings | FLOSS | floss | floss
|
||||
Strings | CyberChef | cyberchef | cyberchef
|
||||
|
||||
Emulation | binee | binee | binee
|
||||
Emulation | capa | capa | capa
|
||||
Emulation | vivbin | vivbin | vivbin
|
||||
|
||||
RE | Ghidra | ghidra,analyzeHeadless | ghidra
|
||||
RE | Cutter | cutter | cutter
|
||||
RE | Radare2 | r2 | r2
|
||||
RE | Rizin | rizin | rizin
|
||||
RE | objdump | objdump | objdump
|
||||
RE | RetDec | retdec-decompiler,retdec-decompiler.py | retdec
|
||||
|
||||
Debuggers | edb | edb | edb
|
||||
Debuggers | gdb | gdb | gdb
|
||||
|
||||
Packers | upx | upx | upx
|
||||
Packers | bytehist | bytehist | bytehist
|
||||
Packers | de4dot | de4dot | de4dot
|
||||
|
||||
Android | apktool | apktool | apktool
|
||||
Android | droidlysis | droidlysis | droidlysis
|
||||
Android | androgui | androgui,androgui.py | androgui
|
||||
Android | baksmali | baksmali | baksmali
|
||||
Android | dex2jar | dex2jar | dex2jar
|
||||
|
||||
Java | CFR | cfr | cfr
|
||||
Java | Procyon | procyon | procyon
|
||||
Java | JAD | jad | jad
|
||||
Java | jd-gui | jd-gui | jd-gui
|
||||
Java | idx_parser | idx_parser,idx_parser.py | idx_parser
|
||||
|
||||
Python | pyinstxtractor | pyinstxtractor,pyinstxtractor.py | pyinstxtractor
|
||||
Python | pycdc | pycdc | pycdc
|
||||
|
||||
JavaScript | js | js | js
|
||||
JavaScript | js-file | js-file | js-file
|
||||
JavaScript | objects.js | objects.js | objects.js
|
||||
JavaScript | box-js | box-js | box-js
|
||||
JavaScript | JSDetox | jsdetox | jsdetox
|
||||
JavaScript | de4js | de4js | de4js
|
||||
|
||||
PowerShell | pwsh | pwsh | pwsh
|
||||
|
||||
Flash | swfdump | swfdump | swfdump
|
||||
Flash | flare | flare | flare
|
||||
Flash | flasm | flasm | flasm
|
||||
Flash | swf_mastah | swf_mastah,swf_mastah.py | swf_mastah
|
||||
Flash | xxxswf | xxxswf | xxxswf
|
||||
|
||||
Shellcode | shellcode2exe | shellcode2exe,shellcode2exe.bat | shellcode2exe
|
||||
Shellcode | scdbg | scdbg | scdbg
|
||||
|
||||
Office | vmonkey | vmonkey | vmonkey
|
||||
Office | pcodedmp | pcodedmp | pcodedmp
|
||||
Office | olevba | olevba | olevba
|
||||
Office | xlmdeobfuscator | xlmdeobfuscator | xlmdeobfuscator
|
||||
Office | oledump | oledump,oledump.py | oledump
|
||||
Office | msoffice-crypt | msoffice-crypt | msoffice-crypt
|
||||
Office | ssview | ssview | ssview
|
||||
|
||||
PDF | pdfid | pdfid,pdfid.py | pdfid, pdfid.py
|
||||
PDF | pdfparser | pdfparser,pdf-parser.py | pdfparser,pdf-parser.py
|
||||
PDF | pdfextract | pdfextract | pdfextract
|
||||
PDF | pdfdecrypt | pdfdecrypt | pdfdecrypt
|
||||
PDF | peepdf | peepdf | peepdf
|
||||
PDF | pdftk | pdftk | pdftk
|
||||
PDF | pdfresurrect | pdfresurrect | pdfresurrect
|
||||
PDF | qpdf | qpdf | qpdf
|
||||
PDF | pdfobjflow | pdfobjflow | pdfobjflow
|
||||
|
||||
RTF | rtfobj | rtfobj | rtfobj
|
||||
RTF | rtfdump | rtfdump,rtfdump.py | rtfdump
|
||||
|
||||
Email | emldump | emldump,emldump.py | emldump
|
||||
Email | msgconvert | msgconvert | msgconvert
|
||||
|
||||
NetMon | Burp Suite | burpsuite | burpsuite
|
||||
NetMon | NetworkMiner | networkminer | networkminer
|
||||
NetMon | polarproxy | polarproxy | polarproxy
|
||||
NetMon | mitmproxy | mitmproxy | mitmproxy
|
||||
NetMon | Wireshark | wireshark | wireshark
|
||||
NetMon | tshark | tshark | tshark
|
||||
NetMon | ngrep | ngrep | ngrep
|
||||
NetMon | tcpxtract | tcpxtract | tcpxtract
|
||||
|
||||
Services | Thug | thug | thug
|
||||
Services | fakedns | fakedns | fakedns
|
||||
Services | fakemail | fakemail | fakemail
|
||||
Services | accept-all-ips | accept-all-ips | accept-all-ips
|
||||
Services | inetsim | inetsim | inetsim
|
||||
Services | fakenet | fakenet | fakenet
|
||||
|
||||
NetUtils | netcat | nc,netcat | nc
|
||||
NetUtils | tor | tor | tor
|
||||
NetUtils | wget | wget | wget
|
||||
NetUtils | curl | curl | curl
|
||||
NetUtils | ssh | ssh | ssh
|
||||
NetUtils | unfurl | unfurl,unfurl_cli.py | unfurl
|
||||
|
||||
Memory | Volatility 2 | vol.py,volatility | volatility
|
||||
Memory | Volatility 3 | vol3,volatility3 | vol3
|
||||
Memory | Rekall | rekall | rekall
|
||||
Memory | linux_mem_diff | linux_mem_diff,linux_mem_diff.py | linux_mem_diff
|
||||
Memory | aeskeyfind | aeskeyfind | aeskeyfind
|
||||
Memory | rsakeyfind | rsakeyfind | rsakeyfind
|
||||
|
||||
Intel | Automater | automater,automater.py | automater
|
||||
Intel | Shodan | shodan | shodan
|
||||
Intel | ipwhois_cli | ipwhois_cli,ipwhois_cli.py | ipwhois_cli
|
||||
Intel | pdnstool | pdnstool | pdnstool
|
||||
Intel | malwoverview | malwoverview,malwoverview.py | malwoverview
|
||||
Intel | nsrllookup | nsrllookup | nsrllookup
|
||||
Intel | vt | vt | vt
|
||||
Intel | YARA engine | yara | yara
|
||||
Intel | Viper | viper | viper
|
||||
Intel | dexray | dexray | dexray
|
||||
Intel | time-decode | time-decode,time-decode.py | time-decode
|
||||
Intel | ioc_writer | ioc_writer,ioc_writer.py | ioc_writer
|
||||
|
||||
Behavior | ltrace | ltrace | ltrace
|
||||
Behavior | strace | strace | strace
|
||||
Behavior | frida | frida,frida-trace | frida
|
||||
Behavior | sysdig | sysdig | sysdig
|
||||
Behavior | unhide | unhide | unhide
|
||||
|
||||
Files | scalpel | scalpel | scalpel
|
||||
Files | unzip | unzip | unzip
|
||||
Files | unrar | unrar | unrar
|
||||
Files | cabextract | cabextract | cabextract
|
||||
|
||||
View | code | code | code
|
||||
View | scite | scite | scite
|
||||
View | xpdf | xpdf | xpdf
|
||||
View | feh | feh | feh
|
||||
View | convert | convert | convert
|
||||
View | tesseract | tesseract | tesseract
|
||||
EOF
|
||||
)
|
||||
|
||||
# Counters
|
||||
missing_cheat=0
|
||||
missing_tldr=0
|
||||
total=0
|
||||
|
||||
# Header
|
||||
print_row "Category" "Tool" "CHEAT" "TLDR"
|
||||
printf -- "%.0s-" $(seq 1 74); echo
|
||||
|
||||
# Iterate tools
|
||||
while IFS='|' read -r category name cheat_list tldr_list; do
|
||||
# skip empty/comment lines
|
||||
[[ -z "${category// }" ]] && continue
|
||||
|
||||
# trim spaces
|
||||
category="${category//[$'\t\r\n']}"; category="$(echo "$category" | sed 's/^ *//; s/ *$//')"
|
||||
name="$(echo "$name" | sed 's/^ *//; s/ *$//')"
|
||||
cheat_list="$(echo "$cheat_list" | sed 's/^ *//; s/ *$//')"
|
||||
tldr_list="$(echo "$tldr_list" | sed 's/^ *//; s/ *$//')"
|
||||
|
||||
(( total++ ))
|
||||
|
||||
# CHEAT check: iterate candidates until one resolves
|
||||
cheat_status="MISS"
|
||||
IFS=',' read -r -a cheat_candidates <<< "$cheat_list"
|
||||
for c in "${cheat_candidates[@]}"; do
|
||||
c="$(echo "$c" | sed 's/^ *//; s/ *$//')"
|
||||
if resolved="$(resolve_cheat_file "$c")"; then
|
||||
cheat_status="OK"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# TLDR check
|
||||
tldr_status="MISS"
|
||||
IFS=',' read -r -a tldr_candidates <<< "$tldr_list"
|
||||
for t in "${tldr_candidates[@]}"; do
|
||||
t="$(echo "$t" | sed 's/^ *//; s/ *$//')"
|
||||
if resolved_tldr="$(resolve_tldr_page "$t")"; then
|
||||
tldr_status="OK"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
[[ "$cheat_status" == "MISS" ]] && ((missing_cheat++))
|
||||
[[ "$tldr_status" == "MISS" ]] && ((missing_tldr++))
|
||||
|
||||
print_row "$category" "$name" "$cheat_status" "$tldr_status"
|
||||
|
||||
done <<< "$TOOL_LIST"
|
||||
|
||||
printf -- "%.0s-" $(seq 1 74); echo
|
||||
printf "Total: %d | Missing CHEAT: %d | Missing TLDR: %d\n" "$total" "$missing_cheat" "$missing_tldr"
|
||||
|
||||
# Hints
|
||||
echo
|
||||
if [[ "$missing_cheat" -gt 0 || "$missing_tldr" -gt 0 ]]; then
|
||||
echo "Hints:"
|
||||
echo "- Add cheat sheets in $CHEAT_DIR_PERSONAL (names without extension recommended)."
|
||||
echo "- Add TLDR pages in: ${TLDR_DIRS[*]} (as <tool>.md)."
|
||||
echo "- fhelp cheat uses resolution similar to this script (handles .py and hyphen/underscore variants)."
|
||||
fi
|
||||
Reference in New Issue
Block a user