diff --git a/projects/go-tools/go/goipgrep/.github/workflows/ci.yml b/projects/go-tools/go/goipgrep/.github/workflows/ci.yml new file mode 100644 index 0000000..24d6d20 --- /dev/null +++ b/projects/go-tools/go/goipgrep/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: ci + +on: + push: + pull_request: + +jobs: + test: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: go test (linux) + if: matrix.os == 'ubuntu-latest' + run: go test ./... + - name: go test (macos) + if: matrix.os == 'macos-latest' + run: go test -ldflags='-linkmode=external' ./... + - run: go vet ./... + - run: ./scripts/build.sh diff --git a/projects/go-tools/go/goipgrep/.gitignore b/projects/go-tools/go/goipgrep/.gitignore new file mode 100644 index 0000000..fe67377 --- /dev/null +++ b/projects/go-tools/go/goipgrep/.gitignore @@ -0,0 +1,4 @@ +dist/ +ipgrep +*.test + diff --git a/projects/go-tools/go/goipgrep/Makefile b/projects/go-tools/go/goipgrep/Makefile new file mode 100644 index 0000000..5ae33bb --- /dev/null +++ b/projects/go-tools/go/goipgrep/Makefile @@ -0,0 +1,22 @@ +.PHONY: build test fmt lint clean + +UNAME_S := $(shell uname -s) +TEST_LDFLAGS := +ifeq ($(UNAME_S),Darwin) +TEST_LDFLAGS := -ldflags='-linkmode=external' +endif + +build: + ./scripts/build.sh + +test: + go test $(TEST_LDFLAGS) ./... + +fmt: + gofmt -w . + +lint: + go vet ./... + +clean: + rm -rf dist diff --git a/projects/go-tools/go/goipgrep/README.md b/projects/go-tools/go/goipgrep/README.md new file mode 100644 index 0000000..42475ee --- /dev/null +++ b/projects/go-tools/go/goipgrep/README.md @@ -0,0 +1,52 @@ +# ipgrep + +Extract IP addresses (and MAC addresses) from text, with optional reachability checks, reverse DNS, and ipinfo.io lookups. + +## Build + +```sh +./scripts/build.sh +./dist/ipgrep --help +``` + +## Install (local) + +```sh +./scripts/build.sh +sudo install -m 0755 ./dist/ipgrep /usr/local/bin/ipgrep +``` + +## Usage + +```sh +# Extract IPv4 addresses from stdin +cat file.txt | ipgrep + +# Extract and uniq/sort +ipgrep -u -s file.txt + +# Extract MAC addresses +ipgrep -m file.txt + +# Reachability filter (auto chooses ICMP if permitted, otherwise TCP probe) +ipgrep -p file.txt + +# Reverse DNS (IP only) +ipgrep -r file.txt + +# Lookup ipinfo.io and output CSV (default header included) +ipgrep -l --format=csv file.txt +``` + +## Notes + +- `-p/--pingable`: + - `--ping-mode=auto` (default) uses ICMP echo when raw sockets are available, otherwise falls back to a TCP connect probe. + - ICMP echo typically requires root or `cap_net_raw` on Linux: + - `sudo setcap cap_net_raw+ep /path/to/ipgrep` +- `-l/--lookup` cache: + - defaults to the OS cache dir (override with `--cache-path`) + - control size and freshness with `--cache-max-entries` and `--cache-ttl` +- MAC `-r/--resolve` and MAC `-p/--pingable` are best-effort and rely on the local neighbor/ARP table: + - Linux: `/proc/net/arp` + - Other OSes: currently unsupported without external tooling diff --git a/projects/go-tools/go/goipgrep/go.mod b/projects/go-tools/go/goipgrep/go.mod new file mode 100644 index 0000000..f8f133d --- /dev/null +++ b/projects/go-tools/go/goipgrep/go.mod @@ -0,0 +1,4 @@ +module git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep + +go 1.22 + diff --git a/projects/go-tools/go/goipgrep/internal/cache/cache.go b/projects/go-tools/go/goipgrep/internal/cache/cache.go new file mode 100644 index 0000000..cc990a3 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/cache/cache.go @@ -0,0 +1,183 @@ +package cache + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/ipinfo" +) + +type entry struct { + Info ipinfo.Info `json:"info"` + FetchedAt time.Time `json:"fetched_at"` +} + +type fileFormat struct { + Version int `json:"version"` + Entries map[string]entry `json:"entries"` +} + +type Store struct { + path string + ttl time.Duration + max int + changed bool + data fileFormat + mu sync.RWMutex +} + +func DefaultPath() string { + dir, err := os.UserCacheDir() + if err != nil { + home, _ := os.UserHomeDir() + if home == "" { + return ".ipgrep.ipinfo.json" + } + return filepath.Join(home, ".cache", "ipgrep", "ipinfo.json") + } + return filepath.Join(dir, "ipgrep", "ipinfo.json") +} + +func Clear(path string) error { + return os.Remove(path) +} + +func Load(path string, ttl time.Duration, maxEntries int) (*Store, error) { + s := &Store{ + path: path, + ttl: ttl, + max: maxEntries, + data: fileFormat{Version: 1, Entries: make(map[string]entry)}, + } + b, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return s, nil + } + return nil, err + } + + // Try v1 format. + var ff fileFormat + if err := json.Unmarshal(b, &ff); err == nil && ff.Version == 1 && ff.Entries != nil { + s.data = ff + return s, nil + } + + // Try legacy format: map[ip]ipinfo.Info. + var legacy map[string]ipinfo.Info + if err := json.Unmarshal(b, &legacy); err == nil && legacy != nil { + now := time.Time{} + for ip, info := range legacy { + s.data.Entries[ip] = entry{Info: info, FetchedAt: now} + } + // Don't mark changed just because we loaded legacy. + return s, nil + } + + return nil, fmt.Errorf("unrecognized cache format: %s", path) +} + +func (s *Store) Get(ip string) (ipinfo.Info, bool) { + s.mu.RLock() + e, ok := s.data.Entries[ip] + s.mu.RUnlock() + if !ok { + return ipinfo.Info{}, false + } + if s.ttl > 0 && !e.FetchedAt.IsZero() && time.Since(e.FetchedAt) > s.ttl { + return ipinfo.Info{}, false + } + return e.Info, true +} + +func (s *Store) Put(ip string, info ipinfo.Info) { + s.mu.Lock() + s.data.Entries[ip] = entry{Info: info, FetchedAt: time.Now().UTC()} + s.changed = true + s.mu.Unlock() +} + +func (s *Store) Changed() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.changed +} + +func (s *Store) Save() error { + if s.path == "" { + return errors.New("cache path is empty") + } + + s.prune() + + s.mu.RLock() + payload := s.data + s.mu.RUnlock() + + dir := filepath.Dir(s.path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + + tmp, err := os.CreateTemp(dir, "ipgrep-cache-*.json") + if err != nil { + return err + } + tmpName := tmp.Name() + enc := json.NewEncoder(tmp) + enc.SetIndent("", " ") + if err := enc.Encode(payload); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, s.path); err != nil { + _ = os.Remove(tmpName) + return err + } + s.mu.Lock() + s.changed = false + s.mu.Unlock() + return nil +} + +func (s *Store) prune() { + if s.max <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if len(s.data.Entries) <= s.max { + return + } + + type kv struct { + k string + t time.Time + } + all := make([]kv, 0, len(s.data.Entries)) + for k, v := range s.data.Entries { + all = append(all, kv{k: k, t: v.FetchedAt}) + } + sort.Slice(all, func(i, j int) bool { + // zero timestamps (legacy) sort oldest. + return all[i].t.Before(all[j].t) + }) + for len(s.data.Entries) > s.max { + k := all[0].k + all = all[1:] + delete(s.data.Entries, k) + s.changed = true + } +} diff --git a/projects/go-tools/go/goipgrep/internal/cache/cache_test.go b/projects/go-tools/go/goipgrep/internal/cache/cache_test.go new file mode 100644 index 0000000..c8fd953 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/cache/cache_test.go @@ -0,0 +1,54 @@ +package cache + +import ( + "os" + "path/filepath" + "testing" + "time" + + "git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/ipinfo" +) + +func TestCacheSaveLoad(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ipinfo.json") + + s, err := Load(path, 0, 0) + if err != nil { + t.Fatalf("Load: %v", err) + } + s.Put("1.2.3.4", ipinfo.Info{IP: "1.2.3.4", Country: "US"}) + if !s.Changed() { + t.Fatalf("expected changed") + } + if err := s.Save(); err != nil { + t.Fatalf("Save: %v", err) + } + + s2, err := Load(path, 0, 0) + if err != nil { + t.Fatalf("Load2: %v", err) + } + info, ok := s2.Get("1.2.3.4") + if !ok || info.Country != "US" { + t.Fatalf("Get: ok=%v info=%#v", ok, info) + } +} + +func TestCacheTTL(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ipinfo.json") + + // Write a v1 cache with an old fetched_at. + b := []byte(`{"version":1,"entries":{"1.2.3.4":{"info":{"ip":"1.2.3.4","country":"US"},"fetched_at":"2000-01-01T00:00:00Z"}}}`) + if err := os.WriteFile(path, b, 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + s, err := Load(path, 24*time.Hour, 0) + if err != nil { + t.Fatalf("Load: %v", err) + } + if _, ok := s.Get("1.2.3.4"); ok { + t.Fatalf("expected TTL miss") + } +} diff --git a/projects/go-tools/go/goipgrep/internal/extract/extract.go b/projects/go-tools/go/goipgrep/internal/extract/extract.go new file mode 100644 index 0000000..2a6f562 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/extract/extract.go @@ -0,0 +1,79 @@ +package extract + +import ( + "bufio" + "context" + "errors" + "io" + "regexp" +) + +type Mode int + +const ( + ModeIP Mode = iota + ModeMAC +) + +type Options struct { + Mode Mode + IPv6 bool +} + +// Grep scans r line-by-line and emits raw matches via emit. +// It avoids reading the full input into memory. +func Grep(ctx context.Context, r io.Reader, opts Options, emit func(string)) error { + if emit == nil { + return errors.New("emit func is nil") + } + + var re *regexp.Regexp + switch opts.Mode { + case ModeMAC: + re = macRE + case ModeIP: + if opts.IPv6 { + re = ipAnyRE + } else { + re = ipv4RE + } + default: + return errors.New("unknown mode") + } + + sc := bufio.NewScanner(r) + // Large-ish buffer for long lines (logs, JSON blobs, etc.). + buf := make([]byte, 64*1024) + sc.Buffer(buf, 1024*1024) + + for sc.Scan() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + line := sc.Bytes() + matches := re.FindAll(line, -1) + for _, m := range matches { + emit(string(m)) + } + } + if err := sc.Err(); err != nil { + return err + } + return nil +} + +// Note: These regexes intentionally find candidates only; callers should +// validate/normalize as needed. +var ( + // \b based candidate matcher, validated later. + ipv4RE = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`) + + // Rough IPv6 candidate matcher (validation happens later if caller chooses to). + ipv6RE = regexp.MustCompile(`(?i)\b(?:[0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}\b`) + + ipAnyRE = regexp.MustCompile(`(?i)(?:` + `\b(?:\d{1,3}\.){3}\d{1,3}\b` + `|` + `\b(?:[0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}\b` + `)`) + + macRE = regexp.MustCompile(`(?i)\b(?:[0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b|\b(?:[0-9a-f]{4}\.){2}[0-9a-f]{4}\b`) +) diff --git a/projects/go-tools/go/goipgrep/internal/extract/extract_test.go b/projects/go-tools/go/goipgrep/internal/extract/extract_test.go new file mode 100644 index 0000000..207eca5 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/extract/extract_test.go @@ -0,0 +1,36 @@ +package extract + +import ( + "context" + "strings" + "testing" +) + +func TestGrepIPv4(t *testing.T) { + in := "a 1.2.3.4 b 999.2.3.4 c\nx 10.0.0.1 y" + var got []string + err := Grep(context.Background(), strings.NewReader(in), Options{Mode: ModeIP}, func(s string) { + got = append(got, s) + }) + if err != nil { + t.Fatalf("Grep: %v", err) + } + // Candidate matcher intentionally returns 999.2.3.4 too; validation happens elsewhere. + if len(got) != 3 { + t.Fatalf("got %d matches: %#v", len(got), got) + } +} + +func TestGrepMAC(t *testing.T) { + in := "aa:bb:cc:dd:ee:ff aabb.ccdd.eeff 00-11-22-33-44-55" + var got []string + err := Grep(context.Background(), strings.NewReader(in), Options{Mode: ModeMAC}, func(s string) { + got = append(got, s) + }) + if err != nil { + t.Fatalf("Grep: %v", err) + } + if len(got) != 3 { + t.Fatalf("got %d matches: %#v", len(got), got) + } +} diff --git a/projects/go-tools/go/goipgrep/internal/ipinfo/ipinfo.go b/projects/go-tools/go/goipgrep/internal/ipinfo/ipinfo.go new file mode 100644 index 0000000..565cdb7 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/ipinfo/ipinfo.go @@ -0,0 +1,131 @@ +package ipinfo + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "runtime" + "sort" + "sync" + "time" +) + +type Info struct { + IP string `json:"ip"` + Hostname string `json:"hostname,omitempty"` + City string `json:"city,omitempty"` + Region string `json:"region,omitempty"` + Country string `json:"country,omitempty"` + Org string `json:"org,omitempty"` +} + +type Client struct { + Token string + Timeout time.Duration +} + +type Cache interface { + Get(ip string) (Info, bool) + Put(ip string, info Info) +} + +func (c Client) Lookup(ctx context.Context, ip string) (Info, error) { + u := fmt.Sprintf("https://ipinfo.io/%s/json", url.PathEscape(ip)) + if c.Token != "" { + u += "?token=" + url.QueryEscape(c.Token) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return Info{}, err + } + req.Header.Set("Accept", "application/json") + + hc := &http.Client{Timeout: c.Timeout} + resp, err := hc.Do(req) + if err != nil { + return Info{}, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return Info{}, fmt.Errorf("ipinfo: %s", resp.Status) + } + var info Info + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return Info{}, err + } + if info.IP == "" { + // Some error payloads decode but don't include the IP. + info.IP = ip + } + return info, nil +} + +func LookupAll(ctx context.Context, ips []string, client Client, store Cache, jobs int) []Info { + if jobs <= 0 { + jobs = runtime.GOMAXPROCS(0) + } + in := make(chan string) + out := make(chan Info) + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for ip := range in { + if store != nil { + if info, ok := store.Get(ip); ok { + select { + case out <- info: + case <-ctx.Done(): + return + } + continue + } + } + + cctx, cancel := context.WithTimeout(ctx, client.Timeout) + info, err := client.Lookup(cctx, ip) + cancel() + if err == nil && info.IP != "" { + if store != nil { + store.Put(ip, info) + } + select { + case out <- info: + case <-ctx.Done(): + return + } + } + } + } + + wg.Add(jobs) + for i := 0; i < jobs; i++ { + go worker() + } + + go func() { + defer close(in) + for _, ip := range ips { + select { + case in <- ip: + case <-ctx.Done(): + return + } + } + }() + + go func() { + wg.Wait() + close(out) + }() + + var res []Info + for info := range out { + res = append(res, info) + } + sort.Slice(res, func(i, j int) bool { return res[i].IP < res[j].IP }) + return res +} diff --git a/projects/go-tools/go/goipgrep/internal/normalize/ip.go b/projects/go-tools/go/goipgrep/internal/normalize/ip.go new file mode 100644 index 0000000..80258e6 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/normalize/ip.go @@ -0,0 +1,27 @@ +package normalize + +import ( + "fmt" + "net" + "strings" +) + +// NormalizeIP validates an IP string and returns a canonical string form. +// If allowIPv6 is false, only IPv4 is accepted. +func NormalizeIP(s string, allowIPv6 bool) (string, bool) { + s = strings.TrimSpace(s) + if s == "" { + return "", false + } + ip := net.ParseIP(s) + if ip == nil { + return "", false + } + if v4 := ip.To4(); v4 != nil { + return fmt.Sprintf("%d.%d.%d.%d", v4[0], v4[1], v4[2], v4[3]), true + } + if !allowIPv6 { + return "", false + } + return ip.String(), true +} diff --git a/projects/go-tools/go/goipgrep/internal/normalize/mac.go b/projects/go-tools/go/goipgrep/internal/normalize/mac.go new file mode 100644 index 0000000..a1f04d8 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/normalize/mac.go @@ -0,0 +1,43 @@ +package normalize + +import ( + "strings" +) + +// NormalizeMAC converts supported MAC forms to canonical lower-case colon-separated form. +// Accepts: xx:xx:xx:xx:xx:xx, xx-xx-xx-xx-xx-xx, xxxx.xxxx.xxxx +func NormalizeMAC(s string) (string, bool) { + s = strings.TrimSpace(s) + if s == "" { + return "", false + } + s = strings.ToLower(s) + + // remove common separators + s = strings.ReplaceAll(s, ":", "") + s = strings.ReplaceAll(s, "-", "") + s = strings.ReplaceAll(s, ".", "") + + if len(s) != 12 { + return "", false + } + for i := 0; i < 12; i++ { + c := s[i] + if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') { + continue + } + return "", false + } + + // canonical form aa:bb:cc:dd:ee:ff + var b strings.Builder + b.Grow(17) + for i := 0; i < 12; i += 2 { + if i > 0 { + b.WriteByte(':') + } + b.WriteByte(s[i]) + b.WriteByte(s[i+1]) + } + return b.String(), true +} diff --git a/projects/go-tools/go/goipgrep/internal/normalize/mac_test.go b/projects/go-tools/go/goipgrep/internal/normalize/mac_test.go new file mode 100644 index 0000000..730534e --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/normalize/mac_test.go @@ -0,0 +1,23 @@ +package normalize + +import "testing" + +func TestNormalizeMAC(t *testing.T) { + cases := map[string]string{ + "AA:BB:CC:DD:EE:FF": "aa:bb:cc:dd:ee:ff", + "aa-bb-cc-dd-ee-ff": "aa:bb:cc:dd:ee:ff", + "aabb.ccdd.eeff": "aa:bb:cc:dd:ee:ff", + } + for in, want := range cases { + got, ok := NormalizeMAC(in) + if !ok { + t.Fatalf("NormalizeMAC(%q) not ok", in) + } + if got != want { + t.Fatalf("NormalizeMAC(%q)=%q want %q", in, got, want) + } + } + if _, ok := NormalizeMAC("not-a-mac"); ok { + t.Fatalf("expected invalid mac") + } +} diff --git a/projects/go-tools/go/goipgrep/internal/ping/ping.go b/projects/go-tools/go/goipgrep/internal/ping/ping.go new file mode 100644 index 0000000..2582304 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/ping/ping.go @@ -0,0 +1,332 @@ +package ping + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/resolve" +) + +type Options struct { + Mode string + Timeout time.Duration + Jobs int +} + +func FilterIPs(ctx context.Context, ips []string, opts Options) ([]string, error) { + if len(ips) == 0 { + return nil, nil + } + mode, warn := selectMode(opts.Mode) + if warn != "" { + fmt.Fprintln(os.Stderr, "ipgrep:", warn) + } + + j := opts.Jobs + if j <= 0 { + j = runtime.GOMAXPROCS(0) + } + + in := make(chan string) + out := make(chan string) + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for ip := range in { + if reachable(ctx, ip, mode, opts.Timeout) { + select { + case out <- ip: + case <-ctx.Done(): + return + } + } + } + } + + wg.Add(j) + for i := 0; i < j; i++ { + go worker() + } + + go func() { + defer close(in) + for _, ip := range ips { + select { + case in <- ip: + case <-ctx.Done(): + return + } + } + }() + + go func() { + wg.Wait() + close(out) + }() + + var res []string + for ip := range out { + res = append(res, ip) + } + return res, nil +} + +// FilterMACs resolves each MAC to one or more local-neighbor IPs then applies IP reachability. +// On non-Linux, neighbor resolution may be unsupported without external tools. +func FilterMACs(ctx context.Context, macs []string, opts Options) ([]string, error) { + tab, err := resolve.LoadNeighborTable() + if err != nil { + return nil, err + } + mode, warn := selectMode(opts.Mode) + if warn != "" { + fmt.Fprintln(os.Stderr, "ipgrep:", warn) + } + + j := opts.Jobs + if j <= 0 { + j = runtime.GOMAXPROCS(0) + } + + in := make(chan string) + out := make(chan string) + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for mac := range in { + ips := tab.ByMAC[mac] + ok := false + for _, ip := range ips { + if reachable(ctx, ip, mode, opts.Timeout) { + ok = true + break + } + } + if ok { + select { + case out <- mac: + case <-ctx.Done(): + return + } + } + } + } + + wg.Add(j) + for i := 0; i < j; i++ { + go worker() + } + + go func() { + defer close(in) + for _, mac := range macs { + select { + case in <- mac: + case <-ctx.Done(): + return + } + } + }() + + go func() { + wg.Wait() + close(out) + }() + + var res []string + for mac := range out { + res = append(res, mac) + } + return res, nil +} + +func selectMode(mode string) (string, string) { + mode = strings.ToLower(strings.TrimSpace(mode)) + switch mode { + case "icmp": + return "icmp", "" + case "tcp": + return "tcp", "" + case "auto", "": + if canICMP() { + return "icmp", "" + } + return "tcp", "ICMP not permitted/available; falling back to TCP probe (use --ping-mode=tcp to silence)" + default: + return "auto", "" + } +} + +func canICMP() bool { + c, err := net.ListenPacket("ip4:icmp", "0.0.0.0") + if err == nil { + _ = c.Close() + return true + } + // Permission failures are common; everything else treat as unavailable too. + return false +} + +func reachable(ctx context.Context, ip string, mode string, timeout time.Duration) bool { + switch mode { + case "icmp": + ok, err := icmpEcho(ctx, ip, timeout) + if err == nil { + return ok + } + // ICMP implementation is IPv4-only; use TCP probe for non-IPv4. + if err != nil && strings.Contains(err.Error(), "not an IPv4") { + return tcpProbe(ctx, ip, timeout) + } + if isPerm(err) { + // If user forced icmp, don't silently fall back. + return false + } + return false + case "tcp": + return tcpProbe(ctx, ip, timeout) + default: + return tcpProbe(ctx, ip, timeout) + } +} + +func icmpEcho(ctx context.Context, dst string, timeout time.Duration) (bool, error) { + ip := net.ParseIP(dst) + if ip == nil || ip.To4() == nil { + return false, errors.New("not an IPv4 address") + } + + c, err := net.ListenPacket("ip4:icmp", "0.0.0.0") + if err != nil { + return false, err + } + defer c.Close() + + // Echo request: type 8, code 0. + id := os.Getpid() & 0xffff + seq := int(time.Now().UnixNano() & 0xffff) + msg := make([]byte, 8+16) + msg[0] = 8 // type + msg[1] = 0 // code + // checksum at [2:4] + msg[4] = byte(id >> 8) + msg[5] = byte(id) + msg[6] = byte(seq >> 8) + msg[7] = byte(seq) + copy(msg[8:], []byte("goipgrep-icmp\000")) + cs := checksum(msg) + msg[2] = byte(cs >> 8) + msg[3] = byte(cs) + + deadline := time.Now().Add(timeout) + _ = c.SetDeadline(deadline) + + dstAddr := &net.IPAddr{IP: ip} + if _, err := c.WriteTo(msg, dstAddr); err != nil { + return false, err + } + + buf := make([]byte, 1500) + for { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + n, _, err := c.ReadFrom(buf) + if err != nil { + return false, err + } + b := buf[:n] + // Some OSes may include IPv4 header; strip if present. + if len(b) >= 20 && (b[0]>>4) == 4 { + ihl := int(b[0]&0x0f) * 4 + if ihl <= len(b) { + b = b[ihl:] + } + } + if len(b) < 8 { + continue + } + typ := b[0] + code := b[1] + if typ != 0 || code != 0 { // echo reply + continue + } + rid := int(b[4])<<8 | int(b[5]) + rseq := int(b[6])<<8 | int(b[7]) + if rid == id && rseq == seq { + return true, nil + } + } +} + +func checksum(b []byte) uint16 { + var sum uint32 + for i := 0; i+1 < len(b); i += 2 { + sum += uint32(b[i])<<8 | uint32(b[i+1]) + } + if len(b)%2 == 1 { + sum += uint32(b[len(b)-1]) << 8 + } + for (sum >> 16) != 0 { + sum = (sum & 0xffff) + (sum >> 16) + } + return ^uint16(sum) +} + +func tcpProbe(ctx context.Context, ip string, timeout time.Duration) bool { + ports := []int{443, 80} + d := net.Dialer{Timeout: timeout} + for _, p := range ports { + addr := net.JoinHostPort(ip, fmt.Sprintf("%d", p)) + c, err := d.DialContext(ctx, "tcp", addr) + if err == nil { + _ = c.Close() + return true + } + if isConnRefused(err) { + return true + } + } + return false +} + +func isPerm(err error) bool { + if err == nil { + return false + } + if errors.Is(err, os.ErrPermission) { + return true + } + var opErr *net.OpError + if errors.As(err, &opErr) { + if errors.Is(opErr.Err, syscall.EPERM) || errors.Is(opErr.Err, syscall.EACCES) { + return true + } + } + return false +} + +func isConnRefused(err error) bool { + var opErr *net.OpError + if errors.As(err, &opErr) { + // On some platforms, opErr.Err is *os.SyscallError wrapping ECONNREFUSED. + var se *os.SyscallError + if errors.As(opErr.Err, &se) { + return errors.Is(se.Err, syscall.ECONNREFUSED) + } + return errors.Is(opErr.Err, syscall.ECONNREFUSED) + } + return false +} diff --git a/projects/go-tools/go/goipgrep/internal/resolve/neigh_linux.go b/projects/go-tools/go/goipgrep/internal/resolve/neigh_linux.go new file mode 100644 index 0000000..4474307 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/resolve/neigh_linux.go @@ -0,0 +1,75 @@ +//go:build linux + +package resolve + +import ( + "bufio" + "io" + "os" + "strings" + + "git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/normalize" +) + +type NeighborTable struct { + ByMAC map[string][]string +} + +func LoadNeighborTable() (*NeighborTable, error) { + f, err := os.Open("/proc/net/arp") + if err != nil { + return nil, err + } + defer f.Close() + return parseProcNetARP(f), nil +} + +func ResolveMACs(macs []string) ([]MACResolution, error) { + tab, err := LoadNeighborTable() + if err != nil { + return nil, err + } + var res []MACResolution + for _, mac := range macs { + res = append(res, MACResolution{MAC: mac, IPs: append([]string(nil), tab.ByMAC[mac]...)}) + } + return res, nil +} + +type MACResolution struct { + MAC string `json:"mac"` + IPs []string `json:"ips,omitempty"` +} + +func parseProcNetARP(r io.Reader) *NeighborTable { + nt := &NeighborTable{ByMAC: make(map[string][]string)} + sc := bufio.NewScanner(r) + first := true + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + if first { + // header + first = false + continue + } + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + ip := fields[0] + macRaw := fields[3] + mac, ok := normalize.NormalizeMAC(macRaw) + if !ok { + continue + } + // Skip incomplete/placeholder entries. + if mac == "00:00:00:00:00:00" { + continue + } + nt.ByMAC[mac] = append(nt.ByMAC[mac], ip) + } + return nt +} diff --git a/projects/go-tools/go/goipgrep/internal/resolve/neigh_unsupported.go b/projects/go-tools/go/goipgrep/internal/resolve/neigh_unsupported.go new file mode 100644 index 0000000..b534f68 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/resolve/neigh_unsupported.go @@ -0,0 +1,23 @@ +//go:build !linux + +package resolve + +import "fmt" + +type NeighborTable struct { + ByMAC map[string][]string +} + +type MACResolution struct { + MAC string `json:"mac"` + IPs []string `json:"ips,omitempty"` +} + +func LoadNeighborTable() (*NeighborTable, error) { + return nil, fmt.Errorf("MAC neighbor/ARP resolution is only supported on Linux without external tools") +} + +func ResolveMACs(macs []string) ([]MACResolution, error) { + _ = macs + return nil, fmt.Errorf("MAC resolution is only supported on Linux without external tools") +} diff --git a/projects/go-tools/go/goipgrep/internal/resolve/reverse.go b/projects/go-tools/go/goipgrep/internal/resolve/reverse.go new file mode 100644 index 0000000..ad555c8 --- /dev/null +++ b/projects/go-tools/go/goipgrep/internal/resolve/reverse.go @@ -0,0 +1,74 @@ +package resolve + +import ( + "context" + "net" + "runtime" + "sort" + "strings" + "sync" + "time" +) + +type ReverseResult struct { + IP string `json:"ip"` + Hostname string `json:"hostname,omitempty"` +} + +func ReverseLookupAll(ctx context.Context, ips []string, timeout time.Duration, jobs int) []ReverseResult { + if jobs <= 0 { + jobs = runtime.GOMAXPROCS(0) + } + + in := make(chan string) + out := make(chan ReverseResult) + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for ip := range in { + r := ReverseResult{IP: ip} + cctx, cancel := context.WithTimeout(ctx, timeout) + names, err := net.DefaultResolver.LookupAddr(cctx, ip) + cancel() + if err == nil && len(names) > 0 { + // trim trailing dot for readability + r.Hostname = strings.TrimSuffix(names[0], ".") + } + select { + case out <- r: + case <-ctx.Done(): + return + } + } + } + + wg.Add(jobs) + for i := 0; i < jobs; i++ { + go worker() + } + + go func() { + defer close(in) + for _, ip := range ips { + select { + case in <- ip: + case <-ctx.Done(): + return + } + } + }() + + go func() { + wg.Wait() + close(out) + }() + + var res []ReverseResult + for r := range out { + res = append(res, r) + } + + sort.Slice(res, func(i, j int) bool { return res[i].IP < res[j].IP }) + return res +} diff --git a/projects/go-tools/go/goipgrep/ipgrep b/projects/go-tools/go/goipgrep/ipgrep deleted file mode 100755 index 3344517..0000000 Binary files a/projects/go-tools/go/goipgrep/ipgrep and /dev/null differ diff --git a/projects/go-tools/go/goipgrep/ipgrep.go b/projects/go-tools/go/goipgrep/ipgrep.go deleted file mode 100644 index 164e008..0000000 --- a/projects/go-tools/go/goipgrep/ipgrep.go +++ /dev/null @@ -1,376 +0,0 @@ -// ipgrep.go -package main - -import ( - "bytes" - "encoding/csv" - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "os/exec" - "regexp" - "sort" - "sync" - "time" -) - -// IPInfo holds the data we want from ipinfo.io -type IPInfo struct { - IP string `json:"ip"` - Hostname string `json:"hostname"` - City string `json:"city"` - Region string `json:"region"` - Country string `json:"country"` - Org string `json:"org"` -} - -func main() { - // Command-line flags. - var ( - sortFlag bool - uniqFlag bool - macFlag bool - pingable bool - resolveFlag bool - lookupFlag bool - fileName string - ) - flag.BoolVar(&uniqFlag, "u", false, "only show uniq IPs/MACs (implies -s)") - flag.BoolVar(&sortFlag, "s", false, "sort output") - flag.BoolVar(&macFlag, "m", false, "grep MAC-IDs instead of IPs") - flag.BoolVar(&pingable, "p", false, "only show 'pingable' entries (MACs still beta)") - flag.BoolVar(&resolveFlag, "r", false, "resolve (uses host for ip and arping for mac)") - flag.BoolVar(&lookupFlag, "l", false, "lookup ip info using ipinfo.io and output CSV: ip,country,region,city,org,hostname") - flag.StringVar(&fileName, "f", "", "input file") - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [-u] [-s] [-m] [-p] [-r] [-l] [-f filename] [file...]\n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() - - // If pingable is set, force sorting and uniqueness. - if pingable || lookupFlag { - sortFlag = true - uniqFlag = true - } - - if lookupFlag && macFlag { - fmt.Fprintln(os.Stderr, "Lookup mode (-l) only works for IP addresses, not MAC addresses.") - os.Exit(1) - } - - // Regular expressions for IPs or MACs. - var pattern string - if macFlag { - // Supports MAC formats: xx:xx:xx:xx:xx:xx or xxxx.xxxx.xxxx - pattern = `(([a-fA-F0-9]{2}[:-]){5}[a-fA-F0-9]{2})|([a-fA-F0-9]{4}\.[a-fA-F0-9]{4}\.[a-fA-F0-9]{4})` - } else { - // Matches valid IPv4 addresses. - pattern = `(((25[0-5])|(2[0-4][0-9])|([0-1]?\d?\d))\.){3}((25[0-5])|(2[0-4][0-9])|([0-1]?\d?\d))` - } - re, err := regexp.Compile(pattern) - if err != nil { - fmt.Fprintf(os.Stderr, "Error compiling regex: %v\n", err) - os.Exit(1) - } - - // Read input from -f file, extra args, or stdin. - var inputData []byte - if fileName != "" { - inputData, err = ioutil.ReadFile(fileName) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading file %s: %v\n", fileName, err) - os.Exit(1) - } - } else if flag.NArg() > 0 { - var buf bytes.Buffer - for _, fname := range flag.Args() { - data, err := ioutil.ReadFile(fname) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading file %s: %v\n", fname, err) - continue - } - buf.Write(data) - buf.WriteByte('\n') - } - inputData = buf.Bytes() - } else { - inputData, err = io.ReadAll(os.Stdin) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading stdin: %v\n", err) - os.Exit(1) - } - } - - // Filter matches using the regex. - matches := re.FindAllString(string(inputData), -1) - if matches == nil { - os.Exit(0) - } - - if sortFlag { - sort.Strings(matches) - } - if uniqFlag { - matches = unique(matches) - } - - if pingable { - matches = filterPingable(matches, macFlag) - if sortFlag { - sort.Strings(matches) - } - } - - // If lookup flag is set, perform ipinfo.io lookups with caching. - if lookupFlag { - cache := loadCache() - var cacheMu sync.Mutex - results := lookupIPInfo(matches, cache, &cacheMu) - - // Sort the results by IP. - sort.Slice(results, func(i, j int) bool { - return results[i].IP < results[j].IP - }) - - // Save the updated cache. - saveCache(cache) - - // Output CSV using csv.Writer for proper CSV formatting. - w := csv.NewWriter(os.Stdout) - // Write header. - if err := w.Write([]string{"ip", "country", "region", "city", "org", "hostname"}); err != nil { - fmt.Fprintf(os.Stderr, "Error writing CSV header: %v\n", err) - os.Exit(1) - } - for _, info := range results { - record := []string{ - info.IP, - info.Country, - info.Region, - info.City, - info.Org, - info.Hostname, - } - if err := w.Write(record); err != nil { - fmt.Fprintf(os.Stderr, "Error writing CSV record: %v\n", err) - os.Exit(1) - } - } - w.Flush() - if err := w.Error(); err != nil { - fmt.Fprintf(os.Stderr, "Error flushing CSV data: %v\n", err) - } - return - } - - // If resolve flag is set, perform resolution. - if resolveFlag { - results := resolveEntries(matches, macFlag) - for _, r := range results { - fmt.Print(r) - if !macFlag { - fmt.Println() - } - } - return - } - - // Otherwise, just output the matches. - for _, m := range matches { - fmt.Println(m) - } -} - -// unique removes duplicate strings from a slice. -func unique(input []string) []string { - seen := make(map[string]struct{}) - var result []string - for _, s := range input { - if _, ok := seen[s]; !ok { - seen[s] = struct{}{} - result = append(result, s) - } - } - return result -} - -// filterPingable runs ping (or arping) concurrently on each entry. -func filterPingable(entries []string, mac bool) []string { - var wg sync.WaitGroup - var mu sync.Mutex - var result []string - - for _, entry := range entries { - wg.Add(1) - go func(e string) { - defer wg.Done() - if isPingable(e, mac) { - mu.Lock() - result = append(result, e) - mu.Unlock() - } - }(entry) - } - wg.Wait() - return result -} - -// isPingable tests if an entry is reachable using ping (or arping for MACs). -func isPingable(entry string, mac bool) bool { - var cmd *exec.Cmd - if mac { - cmd = exec.Command("arping", "-c", "1", "-w", "5000000", entry) - } else { - cmd = exec.Command("ping", "-c", "1", "-w", "1", entry) - } - cmd.Stdout = nil - cmd.Stderr = nil - err := cmd.Run() - return err == nil -} - -// resolveEntries performs resolution via external commands. -func resolveEntries(entries []string, mac bool) []string { - var wg sync.WaitGroup - results := make([]string, len(entries)) - - for i, entry := range entries { - wg.Add(1) - go func(i int, e string) { - defer wg.Done() - if mac { - cmd := exec.Command("arping", "-q", "-c", "1", "-w", "5000000", e) - if err := cmd.Run(); err == nil { - cmd2 := exec.Command("arping", "-c", "1", e) - out, err := cmd2.CombinedOutput() - if err == nil { - results[i] = string(out) - } else { - results[i] = e + "\n" - } - } else { - results[i] = e + "\n" - } - } else { - cmd := exec.Command("host", e) - out, err := cmd.CombinedOutput() - if err == nil { - // Extract the hostname via regex (similar to grep -Po '(?<=pointer ).*') - reHost := regexp.MustCompile(`(?i)(?<=pointer\s)(\S+)`) - match := reHost.FindString(string(out)) - if match != "" { - results[i] = fmt.Sprintf("%s %s", e, match) - } else { - results[i] = e - } - } else { - results[i] = e - } - } - }(i, entry) - } - wg.Wait() - return results -} - -// lookupIPInfo queries ipinfo.io for each IP concurrently, -// checking a local cache before going to the network. -// It returns a slice of IPInfo. -func lookupIPInfo(entries []string, cache map[string]IPInfo, cacheMu *sync.Mutex) []IPInfo { - var wg sync.WaitGroup - var mu sync.Mutex - var results []IPInfo - - client := &http.Client{ - Timeout: 5 * time.Second, - } - - for _, ip := range entries { - wg.Add(1) - go func(ip string) { - defer wg.Done() - // Check cache first. - cacheMu.Lock() - info, found := cache[ip] - cacheMu.Unlock() - if found { - mu.Lock() - results = append(results, info) - mu.Unlock() - return - } - - // Not in cache; perform HTTP lookup. - url := fmt.Sprintf("https://ipinfo.io/%s", ip) - resp, err := client.Get(url) - if err != nil { - return - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return - } - var newInfo IPInfo - if err := json.Unmarshal(body, &newInfo); err != nil { - return - } - // Only add valid responses. - if newInfo.IP == "" { - return - } - // Update cache. - cacheMu.Lock() - cache[ip] = newInfo - cacheMu.Unlock() - mu.Lock() - results = append(results, newInfo) - mu.Unlock() - }(ip) - } - wg.Wait() - return results -} - -// loadCache reads the cache from ~/.ipgrep.db (if present) -// and returns it as a map[string]IPInfo. -func loadCache() map[string]IPInfo { - home, err := os.UserHomeDir() - if err != nil { - return make(map[string]IPInfo) - } - cachePath := home + "/.ipgrep.db" - data, err := ioutil.ReadFile(cachePath) - if err != nil { - // File doesn't exist or can't be read, start with an empty cache. - return make(map[string]IPInfo) - } - var cache map[string]IPInfo - if err := json.Unmarshal(data, &cache); err != nil { - // If unmarshal fails, use an empty cache. - return make(map[string]IPInfo) - } - return cache -} - -// saveCache writes the cache map to ~/.ipgrep.db. -func saveCache(cache map[string]IPInfo) { - home, err := os.UserHomeDir() - if err != nil { - return - } - cachePath := home + "/.ipgrep.db" - data, err := json.MarshalIndent(cache, "", " ") - if err != nil { - fmt.Fprintf(os.Stderr, "Error marshaling cache: %v\n", err) - return - } - if err := ioutil.WriteFile(cachePath, data, 0644); err != nil { - fmt.Fprintf(os.Stderr, "Error writing cache file: %v\n", err) - } -} diff --git a/projects/go-tools/go/goipgrep/scripts/build.sh b/projects/go-tools/go/goipgrep/scripts/build.sh new file mode 100755 index 0000000..3f75242 --- /dev/null +++ b/projects/go-tools/go/goipgrep/scripts/build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env sh +set -eu + +cd "$(dirname "$0")/.." + +mkdir -p dist + +VERSION="${VERSION:-dev}" +COMMIT="${COMMIT:-$(git rev-parse --short HEAD 2>/dev/null || echo unknown)}" +DATE="${DATE:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" + +EXTRA_LDFLAGS="" +GOOS="$(go env GOOS)" +if [ "$GOOS" = "darwin" ]; then + # Some macOS environments treat missing LC_UUID as fatal; external linkmode + # ensures the toolchain emits LC_UUID. + EXTRA_LDFLAGS="-linkmode=external" +fi + +go build \ + -trimpath \ + -ldflags "-s -w ${EXTRA_LDFLAGS} -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" \ + -o dist/ipgrep \ + ./cmd/ipgrep + +echo "built dist/ipgrep"