619b0bc432
- Move single-file tools to tools/ organized by category (security, forensics, data, etc.) - Move multi-file projects to projects/ (go-tools, puzzlebox, timesketch, rust-tools) - Move system scripts to scripts/ (proxy, display, setup, windows) - Organize config files in config/ (shell, visidata, applications) - Move experimental tools to archive/experimental - Create 'what' fuzzy search tool with progressive enhancement (ollama->fzf->grep) - Add initial metadata database for intelligent tool discovery - Preserve git history using 'git mv' commands
42 lines
1.0 KiB
Rust
42 lines
1.0 KiB
Rust
use std::env;
|
|
use std::fs::File;
|
|
use std::io::{self, BufRead, BufReader};
|
|
use std::collections::HashSet;
|
|
use std::hash::{Hash, Hasher};
|
|
use std::collections::hash_map::DefaultHasher;
|
|
|
|
fn main() -> io::Result<()> {
|
|
let args: Vec<String> = env::args().collect();
|
|
|
|
if args.len() > 1 {
|
|
for filename in &args[1..] {
|
|
let file = File::open(filename)?;
|
|
let reader = BufReader::new(file);
|
|
remove_duplicates(reader)?;
|
|
}
|
|
} else {
|
|
// No files provided, read from stdin
|
|
let stdin = io::stdin();
|
|
let reader = stdin.lock();
|
|
remove_duplicates(reader)?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn remove_duplicates<R: BufRead>(reader: R) -> io::Result<()> {
|
|
let mut seen_hashes = HashSet::new();
|
|
for line in reader.lines() {
|
|
let line = line?;
|
|
let mut hasher = DefaultHasher::new();
|
|
line.hash(&mut hasher);
|
|
let hash = hasher.finish();
|
|
|
|
if seen_hashes.insert(hash) {
|
|
println!("{}", line);
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|