619b0bc432
- Move single-file tools to tools/ organized by category (security, forensics, data, etc.) - Move multi-file projects to projects/ (go-tools, puzzlebox, timesketch, rust-tools) - Move system scripts to scripts/ (proxy, display, setup, windows) - Organize config files in config/ (shell, visidata, applications) - Move experimental tools to archive/experimental - Create 'what' fuzzy search tool with progressive enhancement (ollama->fzf->grep) - Add initial metadata database for intelligent tool discovery - Preserve git history using 'git mv' commands
76 lines
1.7 KiB
Go
76 lines
1.7 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"flag"
|
|
"fmt"
|
|
"hash/fnv"
|
|
"log"
|
|
"os"
|
|
)
|
|
|
|
func hashLine(s string) uint32 {
|
|
hasher := fnv.New32a()
|
|
hasher.Write([]byte(s))
|
|
return hasher.Sum32()
|
|
}
|
|
|
|
func main() {
|
|
// Define command line flags
|
|
reverse := flag.Bool("d", false, "Print only lines that appear more than once.")
|
|
help := flag.Bool("h", false, "Display help and usage information.")
|
|
flag.Usage = func() {
|
|
fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
|
|
fmt.Println("This program reads from a file or standard input, deduplicates lines, and outputs the results.")
|
|
fmt.Println("Options:")
|
|
flag.PrintDefaults()
|
|
fmt.Println("Example usage:")
|
|
fmt.Println("\t", os.Args[0], "[options] [filename]")
|
|
fmt.Println("\t", os.Args[0], "-d filename # Only print duplicates")
|
|
fmt.Println("\t", "cat /some/text/file |", os.Args[0], "# Read from standard input")
|
|
}
|
|
flag.Parse()
|
|
|
|
// Check for help flag
|
|
if *help {
|
|
flag.Usage()
|
|
os.Exit(0)
|
|
}
|
|
|
|
// Detemine the input source (file or stdin)
|
|
inputSource := os.Stdin
|
|
var err error
|
|
if flag.NArg() > 0 {
|
|
inputSource, err = os.Open(flag.Args()[0])
|
|
if err != nil {
|
|
log.Fatalf("Failed to open file: %v\n", err)
|
|
}
|
|
defer inputSource.Close()
|
|
}
|
|
seenLines := make(map[uint32]int)
|
|
scanner := bufio.NewScanner(inputSource)
|
|
|
|
//Readin lines
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
hash := hashLine(line)
|
|
seenLines[hash]++
|
|
|
|
if *reverse {
|
|
// Print only lines that appear more than once
|
|
if seenLines[hash] > 1 {
|
|
fmt.Println(line)
|
|
}
|
|
} else {
|
|
// Normal mode, print only unique lines
|
|
if seenLines[hash] == 1 {
|
|
fmt.Println(line)
|
|
}
|
|
}
|
|
}
|
|
//Check for errors during scanning
|
|
if err := scanner.Err(); err != nil {
|
|
log.Fatalf("Failed to read input: %v\n", err)
|
|
}
|
|
}
|