goipgrep: refactor into module; pure-Go ping/resolve; cache+CI; drop binary

This commit is contained in:
tobias
2026-02-17 09:26:30 +01:00
parent 27760b0bf1
commit a931be4707
20 changed files with 1214 additions and 376 deletions

View File

@@ -0,0 +1,26 @@
name: ci
on:
push:
pull_request:
jobs:
test:
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: go test (linux)
if: matrix.os == 'ubuntu-latest'
run: go test ./...
- name: go test (macos)
if: matrix.os == 'macos-latest'
run: go test -ldflags='-linkmode=external' ./...
- run: go vet ./...
- run: ./scripts/build.sh

View File

@@ -0,0 +1,4 @@
dist/
ipgrep
*.test

View File

@@ -0,0 +1,22 @@
.PHONY: build test fmt lint clean
UNAME_S := $(shell uname -s)
TEST_LDFLAGS :=
ifeq ($(UNAME_S),Darwin)
TEST_LDFLAGS := -ldflags='-linkmode=external'
endif
build:
./scripts/build.sh
test:
go test $(TEST_LDFLAGS) ./...
fmt:
gofmt -w .
lint:
go vet ./...
clean:
rm -rf dist

View File

@@ -0,0 +1,52 @@
# ipgrep
Extract IP addresses (and MAC addresses) from text, with optional reachability checks, reverse DNS, and ipinfo.io lookups.
## Build
```sh
./scripts/build.sh
./dist/ipgrep --help
```
## Install (local)
```sh
./scripts/build.sh
sudo install -m 0755 ./dist/ipgrep /usr/local/bin/ipgrep
```
## Usage
```sh
# Extract IPv4 addresses from stdin
cat file.txt | ipgrep
# Extract and uniq/sort
ipgrep -u -s file.txt
# Extract MAC addresses
ipgrep -m file.txt
# Reachability filter (auto chooses ICMP if permitted, otherwise TCP probe)
ipgrep -p file.txt
# Reverse DNS (IP only)
ipgrep -r file.txt
# Lookup ipinfo.io and output CSV (default header included)
ipgrep -l --format=csv file.txt
```
## Notes
- `-p/--pingable`:
- `--ping-mode=auto` (default) uses ICMP echo when raw sockets are available, otherwise falls back to a TCP connect probe.
- ICMP echo typically requires root or `cap_net_raw` on Linux:
- `sudo setcap cap_net_raw+ep /path/to/ipgrep`
- `-l/--lookup` cache:
- defaults to the OS cache dir (override with `--cache-path`)
- control size and freshness with `--cache-max-entries` and `--cache-ttl`
- MAC `-r/--resolve` and MAC `-p/--pingable` are best-effort and rely on the local neighbor/ARP table:
- Linux: `/proc/net/arp`
- Other OSes: currently unsupported without external tooling

View File

@@ -0,0 +1,4 @@
module git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep
go 1.22

View File

@@ -0,0 +1,183 @@
package cache
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"sync"
"time"
"git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/ipinfo"
)
type entry struct {
Info ipinfo.Info `json:"info"`
FetchedAt time.Time `json:"fetched_at"`
}
type fileFormat struct {
Version int `json:"version"`
Entries map[string]entry `json:"entries"`
}
type Store struct {
path string
ttl time.Duration
max int
changed bool
data fileFormat
mu sync.RWMutex
}
func DefaultPath() string {
dir, err := os.UserCacheDir()
if err != nil {
home, _ := os.UserHomeDir()
if home == "" {
return ".ipgrep.ipinfo.json"
}
return filepath.Join(home, ".cache", "ipgrep", "ipinfo.json")
}
return filepath.Join(dir, "ipgrep", "ipinfo.json")
}
func Clear(path string) error {
return os.Remove(path)
}
func Load(path string, ttl time.Duration, maxEntries int) (*Store, error) {
s := &Store{
path: path,
ttl: ttl,
max: maxEntries,
data: fileFormat{Version: 1, Entries: make(map[string]entry)},
}
b, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return s, nil
}
return nil, err
}
// Try v1 format.
var ff fileFormat
if err := json.Unmarshal(b, &ff); err == nil && ff.Version == 1 && ff.Entries != nil {
s.data = ff
return s, nil
}
// Try legacy format: map[ip]ipinfo.Info.
var legacy map[string]ipinfo.Info
if err := json.Unmarshal(b, &legacy); err == nil && legacy != nil {
now := time.Time{}
for ip, info := range legacy {
s.data.Entries[ip] = entry{Info: info, FetchedAt: now}
}
// Don't mark changed just because we loaded legacy.
return s, nil
}
return nil, fmt.Errorf("unrecognized cache format: %s", path)
}
func (s *Store) Get(ip string) (ipinfo.Info, bool) {
s.mu.RLock()
e, ok := s.data.Entries[ip]
s.mu.RUnlock()
if !ok {
return ipinfo.Info{}, false
}
if s.ttl > 0 && !e.FetchedAt.IsZero() && time.Since(e.FetchedAt) > s.ttl {
return ipinfo.Info{}, false
}
return e.Info, true
}
func (s *Store) Put(ip string, info ipinfo.Info) {
s.mu.Lock()
s.data.Entries[ip] = entry{Info: info, FetchedAt: time.Now().UTC()}
s.changed = true
s.mu.Unlock()
}
func (s *Store) Changed() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.changed
}
func (s *Store) Save() error {
if s.path == "" {
return errors.New("cache path is empty")
}
s.prune()
s.mu.RLock()
payload := s.data
s.mu.RUnlock()
dir := filepath.Dir(s.path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, "ipgrep-cache-*.json")
if err != nil {
return err
}
tmpName := tmp.Name()
enc := json.NewEncoder(tmp)
enc.SetIndent("", " ")
if err := enc.Encode(payload); err != nil {
_ = tmp.Close()
_ = os.Remove(tmpName)
return err
}
if err := tmp.Close(); err != nil {
_ = os.Remove(tmpName)
return err
}
if err := os.Rename(tmpName, s.path); err != nil {
_ = os.Remove(tmpName)
return err
}
s.mu.Lock()
s.changed = false
s.mu.Unlock()
return nil
}
func (s *Store) prune() {
if s.max <= 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if len(s.data.Entries) <= s.max {
return
}
type kv struct {
k string
t time.Time
}
all := make([]kv, 0, len(s.data.Entries))
for k, v := range s.data.Entries {
all = append(all, kv{k: k, t: v.FetchedAt})
}
sort.Slice(all, func(i, j int) bool {
// zero timestamps (legacy) sort oldest.
return all[i].t.Before(all[j].t)
})
for len(s.data.Entries) > s.max {
k := all[0].k
all = all[1:]
delete(s.data.Entries, k)
s.changed = true
}
}

View File

@@ -0,0 +1,54 @@
package cache
import (
"os"
"path/filepath"
"testing"
"time"
"git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/ipinfo"
)
func TestCacheSaveLoad(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "ipinfo.json")
s, err := Load(path, 0, 0)
if err != nil {
t.Fatalf("Load: %v", err)
}
s.Put("1.2.3.4", ipinfo.Info{IP: "1.2.3.4", Country: "US"})
if !s.Changed() {
t.Fatalf("expected changed")
}
if err := s.Save(); err != nil {
t.Fatalf("Save: %v", err)
}
s2, err := Load(path, 0, 0)
if err != nil {
t.Fatalf("Load2: %v", err)
}
info, ok := s2.Get("1.2.3.4")
if !ok || info.Country != "US" {
t.Fatalf("Get: ok=%v info=%#v", ok, info)
}
}
func TestCacheTTL(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "ipinfo.json")
// Write a v1 cache with an old fetched_at.
b := []byte(`{"version":1,"entries":{"1.2.3.4":{"info":{"ip":"1.2.3.4","country":"US"},"fetched_at":"2000-01-01T00:00:00Z"}}}`)
if err := os.WriteFile(path, b, 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
s, err := Load(path, 24*time.Hour, 0)
if err != nil {
t.Fatalf("Load: %v", err)
}
if _, ok := s.Get("1.2.3.4"); ok {
t.Fatalf("expected TTL miss")
}
}

View File

@@ -0,0 +1,79 @@
package extract
import (
"bufio"
"context"
"errors"
"io"
"regexp"
)
type Mode int
const (
ModeIP Mode = iota
ModeMAC
)
type Options struct {
Mode Mode
IPv6 bool
}
// Grep scans r line-by-line and emits raw matches via emit.
// It avoids reading the full input into memory.
func Grep(ctx context.Context, r io.Reader, opts Options, emit func(string)) error {
if emit == nil {
return errors.New("emit func is nil")
}
var re *regexp.Regexp
switch opts.Mode {
case ModeMAC:
re = macRE
case ModeIP:
if opts.IPv6 {
re = ipAnyRE
} else {
re = ipv4RE
}
default:
return errors.New("unknown mode")
}
sc := bufio.NewScanner(r)
// Large-ish buffer for long lines (logs, JSON blobs, etc.).
buf := make([]byte, 64*1024)
sc.Buffer(buf, 1024*1024)
for sc.Scan() {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
line := sc.Bytes()
matches := re.FindAll(line, -1)
for _, m := range matches {
emit(string(m))
}
}
if err := sc.Err(); err != nil {
return err
}
return nil
}
// Note: These regexes intentionally find candidates only; callers should
// validate/normalize as needed.
var (
// \b based candidate matcher, validated later.
ipv4RE = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
// Rough IPv6 candidate matcher (validation happens later if caller chooses to).
ipv6RE = regexp.MustCompile(`(?i)\b(?:[0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}\b`)
ipAnyRE = regexp.MustCompile(`(?i)(?:` + `\b(?:\d{1,3}\.){3}\d{1,3}\b` + `|` + `\b(?:[0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}\b` + `)`)
macRE = regexp.MustCompile(`(?i)\b(?:[0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b|\b(?:[0-9a-f]{4}\.){2}[0-9a-f]{4}\b`)
)

View File

@@ -0,0 +1,36 @@
package extract
import (
"context"
"strings"
"testing"
)
func TestGrepIPv4(t *testing.T) {
in := "a 1.2.3.4 b 999.2.3.4 c\nx 10.0.0.1 y"
var got []string
err := Grep(context.Background(), strings.NewReader(in), Options{Mode: ModeIP}, func(s string) {
got = append(got, s)
})
if err != nil {
t.Fatalf("Grep: %v", err)
}
// Candidate matcher intentionally returns 999.2.3.4 too; validation happens elsewhere.
if len(got) != 3 {
t.Fatalf("got %d matches: %#v", len(got), got)
}
}
func TestGrepMAC(t *testing.T) {
in := "aa:bb:cc:dd:ee:ff aabb.ccdd.eeff 00-11-22-33-44-55"
var got []string
err := Grep(context.Background(), strings.NewReader(in), Options{Mode: ModeMAC}, func(s string) {
got = append(got, s)
})
if err != nil {
t.Fatalf("Grep: %v", err)
}
if len(got) != 3 {
t.Fatalf("got %d matches: %#v", len(got), got)
}
}

View File

@@ -0,0 +1,131 @@
package ipinfo
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"runtime"
"sort"
"sync"
"time"
)
type Info struct {
IP string `json:"ip"`
Hostname string `json:"hostname,omitempty"`
City string `json:"city,omitempty"`
Region string `json:"region,omitempty"`
Country string `json:"country,omitempty"`
Org string `json:"org,omitempty"`
}
type Client struct {
Token string
Timeout time.Duration
}
type Cache interface {
Get(ip string) (Info, bool)
Put(ip string, info Info)
}
func (c Client) Lookup(ctx context.Context, ip string) (Info, error) {
u := fmt.Sprintf("https://ipinfo.io/%s/json", url.PathEscape(ip))
if c.Token != "" {
u += "?token=" + url.QueryEscape(c.Token)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return Info{}, err
}
req.Header.Set("Accept", "application/json")
hc := &http.Client{Timeout: c.Timeout}
resp, err := hc.Do(req)
if err != nil {
return Info{}, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return Info{}, fmt.Errorf("ipinfo: %s", resp.Status)
}
var info Info
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
return Info{}, err
}
if info.IP == "" {
// Some error payloads decode but don't include the IP.
info.IP = ip
}
return info, nil
}
func LookupAll(ctx context.Context, ips []string, client Client, store Cache, jobs int) []Info {
if jobs <= 0 {
jobs = runtime.GOMAXPROCS(0)
}
in := make(chan string)
out := make(chan Info)
var wg sync.WaitGroup
worker := func() {
defer wg.Done()
for ip := range in {
if store != nil {
if info, ok := store.Get(ip); ok {
select {
case out <- info:
case <-ctx.Done():
return
}
continue
}
}
cctx, cancel := context.WithTimeout(ctx, client.Timeout)
info, err := client.Lookup(cctx, ip)
cancel()
if err == nil && info.IP != "" {
if store != nil {
store.Put(ip, info)
}
select {
case out <- info:
case <-ctx.Done():
return
}
}
}
}
wg.Add(jobs)
for i := 0; i < jobs; i++ {
go worker()
}
go func() {
defer close(in)
for _, ip := range ips {
select {
case in <- ip:
case <-ctx.Done():
return
}
}
}()
go func() {
wg.Wait()
close(out)
}()
var res []Info
for info := range out {
res = append(res, info)
}
sort.Slice(res, func(i, j int) bool { return res[i].IP < res[j].IP })
return res
}

View File

@@ -0,0 +1,27 @@
package normalize
import (
"fmt"
"net"
"strings"
)
// NormalizeIP validates an IP string and returns a canonical string form.
// If allowIPv6 is false, only IPv4 is accepted.
func NormalizeIP(s string, allowIPv6 bool) (string, bool) {
s = strings.TrimSpace(s)
if s == "" {
return "", false
}
ip := net.ParseIP(s)
if ip == nil {
return "", false
}
if v4 := ip.To4(); v4 != nil {
return fmt.Sprintf("%d.%d.%d.%d", v4[0], v4[1], v4[2], v4[3]), true
}
if !allowIPv6 {
return "", false
}
return ip.String(), true
}

View File

@@ -0,0 +1,43 @@
package normalize
import (
"strings"
)
// NormalizeMAC converts supported MAC forms to canonical lower-case colon-separated form.
// Accepts: xx:xx:xx:xx:xx:xx, xx-xx-xx-xx-xx-xx, xxxx.xxxx.xxxx
func NormalizeMAC(s string) (string, bool) {
s = strings.TrimSpace(s)
if s == "" {
return "", false
}
s = strings.ToLower(s)
// remove common separators
s = strings.ReplaceAll(s, ":", "")
s = strings.ReplaceAll(s, "-", "")
s = strings.ReplaceAll(s, ".", "")
if len(s) != 12 {
return "", false
}
for i := 0; i < 12; i++ {
c := s[i]
if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') {
continue
}
return "", false
}
// canonical form aa:bb:cc:dd:ee:ff
var b strings.Builder
b.Grow(17)
for i := 0; i < 12; i += 2 {
if i > 0 {
b.WriteByte(':')
}
b.WriteByte(s[i])
b.WriteByte(s[i+1])
}
return b.String(), true
}

View File

@@ -0,0 +1,23 @@
package normalize
import "testing"
func TestNormalizeMAC(t *testing.T) {
cases := map[string]string{
"AA:BB:CC:DD:EE:FF": "aa:bb:cc:dd:ee:ff",
"aa-bb-cc-dd-ee-ff": "aa:bb:cc:dd:ee:ff",
"aabb.ccdd.eeff": "aa:bb:cc:dd:ee:ff",
}
for in, want := range cases {
got, ok := NormalizeMAC(in)
if !ok {
t.Fatalf("NormalizeMAC(%q) not ok", in)
}
if got != want {
t.Fatalf("NormalizeMAC(%q)=%q want %q", in, got, want)
}
}
if _, ok := NormalizeMAC("not-a-mac"); ok {
t.Fatalf("expected invalid mac")
}
}

View File

@@ -0,0 +1,332 @@
package ping
import (
"context"
"errors"
"fmt"
"net"
"os"
"runtime"
"strings"
"sync"
"syscall"
"time"
"git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/resolve"
)
type Options struct {
Mode string
Timeout time.Duration
Jobs int
}
func FilterIPs(ctx context.Context, ips []string, opts Options) ([]string, error) {
if len(ips) == 0 {
return nil, nil
}
mode, warn := selectMode(opts.Mode)
if warn != "" {
fmt.Fprintln(os.Stderr, "ipgrep:", warn)
}
j := opts.Jobs
if j <= 0 {
j = runtime.GOMAXPROCS(0)
}
in := make(chan string)
out := make(chan string)
var wg sync.WaitGroup
worker := func() {
defer wg.Done()
for ip := range in {
if reachable(ctx, ip, mode, opts.Timeout) {
select {
case out <- ip:
case <-ctx.Done():
return
}
}
}
}
wg.Add(j)
for i := 0; i < j; i++ {
go worker()
}
go func() {
defer close(in)
for _, ip := range ips {
select {
case in <- ip:
case <-ctx.Done():
return
}
}
}()
go func() {
wg.Wait()
close(out)
}()
var res []string
for ip := range out {
res = append(res, ip)
}
return res, nil
}
// FilterMACs resolves each MAC to one or more local-neighbor IPs then applies IP reachability.
// On non-Linux, neighbor resolution may be unsupported without external tools.
func FilterMACs(ctx context.Context, macs []string, opts Options) ([]string, error) {
tab, err := resolve.LoadNeighborTable()
if err != nil {
return nil, err
}
mode, warn := selectMode(opts.Mode)
if warn != "" {
fmt.Fprintln(os.Stderr, "ipgrep:", warn)
}
j := opts.Jobs
if j <= 0 {
j = runtime.GOMAXPROCS(0)
}
in := make(chan string)
out := make(chan string)
var wg sync.WaitGroup
worker := func() {
defer wg.Done()
for mac := range in {
ips := tab.ByMAC[mac]
ok := false
for _, ip := range ips {
if reachable(ctx, ip, mode, opts.Timeout) {
ok = true
break
}
}
if ok {
select {
case out <- mac:
case <-ctx.Done():
return
}
}
}
}
wg.Add(j)
for i := 0; i < j; i++ {
go worker()
}
go func() {
defer close(in)
for _, mac := range macs {
select {
case in <- mac:
case <-ctx.Done():
return
}
}
}()
go func() {
wg.Wait()
close(out)
}()
var res []string
for mac := range out {
res = append(res, mac)
}
return res, nil
}
func selectMode(mode string) (string, string) {
mode = strings.ToLower(strings.TrimSpace(mode))
switch mode {
case "icmp":
return "icmp", ""
case "tcp":
return "tcp", ""
case "auto", "":
if canICMP() {
return "icmp", ""
}
return "tcp", "ICMP not permitted/available; falling back to TCP probe (use --ping-mode=tcp to silence)"
default:
return "auto", ""
}
}
func canICMP() bool {
c, err := net.ListenPacket("ip4:icmp", "0.0.0.0")
if err == nil {
_ = c.Close()
return true
}
// Permission failures are common; everything else treat as unavailable too.
return false
}
func reachable(ctx context.Context, ip string, mode string, timeout time.Duration) bool {
switch mode {
case "icmp":
ok, err := icmpEcho(ctx, ip, timeout)
if err == nil {
return ok
}
// ICMP implementation is IPv4-only; use TCP probe for non-IPv4.
if err != nil && strings.Contains(err.Error(), "not an IPv4") {
return tcpProbe(ctx, ip, timeout)
}
if isPerm(err) {
// If user forced icmp, don't silently fall back.
return false
}
return false
case "tcp":
return tcpProbe(ctx, ip, timeout)
default:
return tcpProbe(ctx, ip, timeout)
}
}
func icmpEcho(ctx context.Context, dst string, timeout time.Duration) (bool, error) {
ip := net.ParseIP(dst)
if ip == nil || ip.To4() == nil {
return false, errors.New("not an IPv4 address")
}
c, err := net.ListenPacket("ip4:icmp", "0.0.0.0")
if err != nil {
return false, err
}
defer c.Close()
// Echo request: type 8, code 0.
id := os.Getpid() & 0xffff
seq := int(time.Now().UnixNano() & 0xffff)
msg := make([]byte, 8+16)
msg[0] = 8 // type
msg[1] = 0 // code
// checksum at [2:4]
msg[4] = byte(id >> 8)
msg[5] = byte(id)
msg[6] = byte(seq >> 8)
msg[7] = byte(seq)
copy(msg[8:], []byte("goipgrep-icmp\000"))
cs := checksum(msg)
msg[2] = byte(cs >> 8)
msg[3] = byte(cs)
deadline := time.Now().Add(timeout)
_ = c.SetDeadline(deadline)
dstAddr := &net.IPAddr{IP: ip}
if _, err := c.WriteTo(msg, dstAddr); err != nil {
return false, err
}
buf := make([]byte, 1500)
for {
select {
case <-ctx.Done():
return false, ctx.Err()
default:
}
n, _, err := c.ReadFrom(buf)
if err != nil {
return false, err
}
b := buf[:n]
// Some OSes may include IPv4 header; strip if present.
if len(b) >= 20 && (b[0]>>4) == 4 {
ihl := int(b[0]&0x0f) * 4
if ihl <= len(b) {
b = b[ihl:]
}
}
if len(b) < 8 {
continue
}
typ := b[0]
code := b[1]
if typ != 0 || code != 0 { // echo reply
continue
}
rid := int(b[4])<<8 | int(b[5])
rseq := int(b[6])<<8 | int(b[7])
if rid == id && rseq == seq {
return true, nil
}
}
}
func checksum(b []byte) uint16 {
var sum uint32
for i := 0; i+1 < len(b); i += 2 {
sum += uint32(b[i])<<8 | uint32(b[i+1])
}
if len(b)%2 == 1 {
sum += uint32(b[len(b)-1]) << 8
}
for (sum >> 16) != 0 {
sum = (sum & 0xffff) + (sum >> 16)
}
return ^uint16(sum)
}
func tcpProbe(ctx context.Context, ip string, timeout time.Duration) bool {
ports := []int{443, 80}
d := net.Dialer{Timeout: timeout}
for _, p := range ports {
addr := net.JoinHostPort(ip, fmt.Sprintf("%d", p))
c, err := d.DialContext(ctx, "tcp", addr)
if err == nil {
_ = c.Close()
return true
}
if isConnRefused(err) {
return true
}
}
return false
}
func isPerm(err error) bool {
if err == nil {
return false
}
if errors.Is(err, os.ErrPermission) {
return true
}
var opErr *net.OpError
if errors.As(err, &opErr) {
if errors.Is(opErr.Err, syscall.EPERM) || errors.Is(opErr.Err, syscall.EACCES) {
return true
}
}
return false
}
func isConnRefused(err error) bool {
var opErr *net.OpError
if errors.As(err, &opErr) {
// On some platforms, opErr.Err is *os.SyscallError wrapping ECONNREFUSED.
var se *os.SyscallError
if errors.As(opErr.Err, &se) {
return errors.Is(se.Err, syscall.ECONNREFUSED)
}
return errors.Is(opErr.Err, syscall.ECONNREFUSED)
}
return false
}

View File

@@ -0,0 +1,75 @@
//go:build linux
package resolve
import (
"bufio"
"io"
"os"
"strings"
"git.ktf.ninja/tabledevil/gists/projects/go-tools/go/goipgrep/internal/normalize"
)
type NeighborTable struct {
ByMAC map[string][]string
}
func LoadNeighborTable() (*NeighborTable, error) {
f, err := os.Open("/proc/net/arp")
if err != nil {
return nil, err
}
defer f.Close()
return parseProcNetARP(f), nil
}
func ResolveMACs(macs []string) ([]MACResolution, error) {
tab, err := LoadNeighborTable()
if err != nil {
return nil, err
}
var res []MACResolution
for _, mac := range macs {
res = append(res, MACResolution{MAC: mac, IPs: append([]string(nil), tab.ByMAC[mac]...)})
}
return res, nil
}
type MACResolution struct {
MAC string `json:"mac"`
IPs []string `json:"ips,omitempty"`
}
func parseProcNetARP(r io.Reader) *NeighborTable {
nt := &NeighborTable{ByMAC: make(map[string][]string)}
sc := bufio.NewScanner(r)
first := true
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
if first {
// header
first = false
continue
}
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
ip := fields[0]
macRaw := fields[3]
mac, ok := normalize.NormalizeMAC(macRaw)
if !ok {
continue
}
// Skip incomplete/placeholder entries.
if mac == "00:00:00:00:00:00" {
continue
}
nt.ByMAC[mac] = append(nt.ByMAC[mac], ip)
}
return nt
}

View File

@@ -0,0 +1,23 @@
//go:build !linux
package resolve
import "fmt"
type NeighborTable struct {
ByMAC map[string][]string
}
type MACResolution struct {
MAC string `json:"mac"`
IPs []string `json:"ips,omitempty"`
}
func LoadNeighborTable() (*NeighborTable, error) {
return nil, fmt.Errorf("MAC neighbor/ARP resolution is only supported on Linux without external tools")
}
func ResolveMACs(macs []string) ([]MACResolution, error) {
_ = macs
return nil, fmt.Errorf("MAC resolution is only supported on Linux without external tools")
}

View File

@@ -0,0 +1,74 @@
package resolve
import (
"context"
"net"
"runtime"
"sort"
"strings"
"sync"
"time"
)
type ReverseResult struct {
IP string `json:"ip"`
Hostname string `json:"hostname,omitempty"`
}
func ReverseLookupAll(ctx context.Context, ips []string, timeout time.Duration, jobs int) []ReverseResult {
if jobs <= 0 {
jobs = runtime.GOMAXPROCS(0)
}
in := make(chan string)
out := make(chan ReverseResult)
var wg sync.WaitGroup
worker := func() {
defer wg.Done()
for ip := range in {
r := ReverseResult{IP: ip}
cctx, cancel := context.WithTimeout(ctx, timeout)
names, err := net.DefaultResolver.LookupAddr(cctx, ip)
cancel()
if err == nil && len(names) > 0 {
// trim trailing dot for readability
r.Hostname = strings.TrimSuffix(names[0], ".")
}
select {
case out <- r:
case <-ctx.Done():
return
}
}
}
wg.Add(jobs)
for i := 0; i < jobs; i++ {
go worker()
}
go func() {
defer close(in)
for _, ip := range ips {
select {
case in <- ip:
case <-ctx.Done():
return
}
}
}()
go func() {
wg.Wait()
close(out)
}()
var res []ReverseResult
for r := range out {
res = append(res, r)
}
sort.Slice(res, func(i, j int) bool { return res[i].IP < res[j].IP })
return res
}

Binary file not shown.

View File

@@ -1,376 +0,0 @@
// ipgrep.go
package main
import (
"bytes"
"encoding/csv"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"regexp"
"sort"
"sync"
"time"
)
// IPInfo holds the data we want from ipinfo.io
type IPInfo struct {
IP string `json:"ip"`
Hostname string `json:"hostname"`
City string `json:"city"`
Region string `json:"region"`
Country string `json:"country"`
Org string `json:"org"`
}
func main() {
// Command-line flags.
var (
sortFlag bool
uniqFlag bool
macFlag bool
pingable bool
resolveFlag bool
lookupFlag bool
fileName string
)
flag.BoolVar(&uniqFlag, "u", false, "only show uniq IPs/MACs (implies -s)")
flag.BoolVar(&sortFlag, "s", false, "sort output")
flag.BoolVar(&macFlag, "m", false, "grep MAC-IDs instead of IPs")
flag.BoolVar(&pingable, "p", false, "only show 'pingable' entries (MACs still beta)")
flag.BoolVar(&resolveFlag, "r", false, "resolve (uses host for ip and arping for mac)")
flag.BoolVar(&lookupFlag, "l", false, "lookup ip info using ipinfo.io and output CSV: ip,country,region,city,org,hostname")
flag.StringVar(&fileName, "f", "", "input file")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [-u] [-s] [-m] [-p] [-r] [-l] [-f filename] [file...]\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
// If pingable is set, force sorting and uniqueness.
if pingable || lookupFlag {
sortFlag = true
uniqFlag = true
}
if lookupFlag && macFlag {
fmt.Fprintln(os.Stderr, "Lookup mode (-l) only works for IP addresses, not MAC addresses.")
os.Exit(1)
}
// Regular expressions for IPs or MACs.
var pattern string
if macFlag {
// Supports MAC formats: xx:xx:xx:xx:xx:xx or xxxx.xxxx.xxxx
pattern = `(([a-fA-F0-9]{2}[:-]){5}[a-fA-F0-9]{2})|([a-fA-F0-9]{4}\.[a-fA-F0-9]{4}\.[a-fA-F0-9]{4})`
} else {
// Matches valid IPv4 addresses.
pattern = `(((25[0-5])|(2[0-4][0-9])|([0-1]?\d?\d))\.){3}((25[0-5])|(2[0-4][0-9])|([0-1]?\d?\d))`
}
re, err := regexp.Compile(pattern)
if err != nil {
fmt.Fprintf(os.Stderr, "Error compiling regex: %v\n", err)
os.Exit(1)
}
// Read input from -f file, extra args, or stdin.
var inputData []byte
if fileName != "" {
inputData, err = ioutil.ReadFile(fileName)
if err != nil {
fmt.Fprintf(os.Stderr, "Error reading file %s: %v\n", fileName, err)
os.Exit(1)
}
} else if flag.NArg() > 0 {
var buf bytes.Buffer
for _, fname := range flag.Args() {
data, err := ioutil.ReadFile(fname)
if err != nil {
fmt.Fprintf(os.Stderr, "Error reading file %s: %v\n", fname, err)
continue
}
buf.Write(data)
buf.WriteByte('\n')
}
inputData = buf.Bytes()
} else {
inputData, err = io.ReadAll(os.Stdin)
if err != nil {
fmt.Fprintf(os.Stderr, "Error reading stdin: %v\n", err)
os.Exit(1)
}
}
// Filter matches using the regex.
matches := re.FindAllString(string(inputData), -1)
if matches == nil {
os.Exit(0)
}
if sortFlag {
sort.Strings(matches)
}
if uniqFlag {
matches = unique(matches)
}
if pingable {
matches = filterPingable(matches, macFlag)
if sortFlag {
sort.Strings(matches)
}
}
// If lookup flag is set, perform ipinfo.io lookups with caching.
if lookupFlag {
cache := loadCache()
var cacheMu sync.Mutex
results := lookupIPInfo(matches, cache, &cacheMu)
// Sort the results by IP.
sort.Slice(results, func(i, j int) bool {
return results[i].IP < results[j].IP
})
// Save the updated cache.
saveCache(cache)
// Output CSV using csv.Writer for proper CSV formatting.
w := csv.NewWriter(os.Stdout)
// Write header.
if err := w.Write([]string{"ip", "country", "region", "city", "org", "hostname"}); err != nil {
fmt.Fprintf(os.Stderr, "Error writing CSV header: %v\n", err)
os.Exit(1)
}
for _, info := range results {
record := []string{
info.IP,
info.Country,
info.Region,
info.City,
info.Org,
info.Hostname,
}
if err := w.Write(record); err != nil {
fmt.Fprintf(os.Stderr, "Error writing CSV record: %v\n", err)
os.Exit(1)
}
}
w.Flush()
if err := w.Error(); err != nil {
fmt.Fprintf(os.Stderr, "Error flushing CSV data: %v\n", err)
}
return
}
// If resolve flag is set, perform resolution.
if resolveFlag {
results := resolveEntries(matches, macFlag)
for _, r := range results {
fmt.Print(r)
if !macFlag {
fmt.Println()
}
}
return
}
// Otherwise, just output the matches.
for _, m := range matches {
fmt.Println(m)
}
}
// unique removes duplicate strings from a slice.
func unique(input []string) []string {
seen := make(map[string]struct{})
var result []string
for _, s := range input {
if _, ok := seen[s]; !ok {
seen[s] = struct{}{}
result = append(result, s)
}
}
return result
}
// filterPingable runs ping (or arping) concurrently on each entry.
func filterPingable(entries []string, mac bool) []string {
var wg sync.WaitGroup
var mu sync.Mutex
var result []string
for _, entry := range entries {
wg.Add(1)
go func(e string) {
defer wg.Done()
if isPingable(e, mac) {
mu.Lock()
result = append(result, e)
mu.Unlock()
}
}(entry)
}
wg.Wait()
return result
}
// isPingable tests if an entry is reachable using ping (or arping for MACs).
func isPingable(entry string, mac bool) bool {
var cmd *exec.Cmd
if mac {
cmd = exec.Command("arping", "-c", "1", "-w", "5000000", entry)
} else {
cmd = exec.Command("ping", "-c", "1", "-w", "1", entry)
}
cmd.Stdout = nil
cmd.Stderr = nil
err := cmd.Run()
return err == nil
}
// resolveEntries performs resolution via external commands.
func resolveEntries(entries []string, mac bool) []string {
var wg sync.WaitGroup
results := make([]string, len(entries))
for i, entry := range entries {
wg.Add(1)
go func(i int, e string) {
defer wg.Done()
if mac {
cmd := exec.Command("arping", "-q", "-c", "1", "-w", "5000000", e)
if err := cmd.Run(); err == nil {
cmd2 := exec.Command("arping", "-c", "1", e)
out, err := cmd2.CombinedOutput()
if err == nil {
results[i] = string(out)
} else {
results[i] = e + "\n"
}
} else {
results[i] = e + "\n"
}
} else {
cmd := exec.Command("host", e)
out, err := cmd.CombinedOutput()
if err == nil {
// Extract the hostname via regex (similar to grep -Po '(?<=pointer ).*')
reHost := regexp.MustCompile(`(?i)(?<=pointer\s)(\S+)`)
match := reHost.FindString(string(out))
if match != "" {
results[i] = fmt.Sprintf("%s %s", e, match)
} else {
results[i] = e
}
} else {
results[i] = e
}
}
}(i, entry)
}
wg.Wait()
return results
}
// lookupIPInfo queries ipinfo.io for each IP concurrently,
// checking a local cache before going to the network.
// It returns a slice of IPInfo.
func lookupIPInfo(entries []string, cache map[string]IPInfo, cacheMu *sync.Mutex) []IPInfo {
var wg sync.WaitGroup
var mu sync.Mutex
var results []IPInfo
client := &http.Client{
Timeout: 5 * time.Second,
}
for _, ip := range entries {
wg.Add(1)
go func(ip string) {
defer wg.Done()
// Check cache first.
cacheMu.Lock()
info, found := cache[ip]
cacheMu.Unlock()
if found {
mu.Lock()
results = append(results, info)
mu.Unlock()
return
}
// Not in cache; perform HTTP lookup.
url := fmt.Sprintf("https://ipinfo.io/%s", ip)
resp, err := client.Get(url)
if err != nil {
return
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return
}
var newInfo IPInfo
if err := json.Unmarshal(body, &newInfo); err != nil {
return
}
// Only add valid responses.
if newInfo.IP == "" {
return
}
// Update cache.
cacheMu.Lock()
cache[ip] = newInfo
cacheMu.Unlock()
mu.Lock()
results = append(results, newInfo)
mu.Unlock()
}(ip)
}
wg.Wait()
return results
}
// loadCache reads the cache from ~/.ipgrep.db (if present)
// and returns it as a map[string]IPInfo.
func loadCache() map[string]IPInfo {
home, err := os.UserHomeDir()
if err != nil {
return make(map[string]IPInfo)
}
cachePath := home + "/.ipgrep.db"
data, err := ioutil.ReadFile(cachePath)
if err != nil {
// File doesn't exist or can't be read, start with an empty cache.
return make(map[string]IPInfo)
}
var cache map[string]IPInfo
if err := json.Unmarshal(data, &cache); err != nil {
// If unmarshal fails, use an empty cache.
return make(map[string]IPInfo)
}
return cache
}
// saveCache writes the cache map to ~/.ipgrep.db.
func saveCache(cache map[string]IPInfo) {
home, err := os.UserHomeDir()
if err != nil {
return
}
cachePath := home + "/.ipgrep.db"
data, err := json.MarshalIndent(cache, "", " ")
if err != nil {
fmt.Fprintf(os.Stderr, "Error marshaling cache: %v\n", err)
return
}
if err := ioutil.WriteFile(cachePath, data, 0644); err != nil {
fmt.Fprintf(os.Stderr, "Error writing cache file: %v\n", err)
}
}

View File

@@ -0,0 +1,26 @@
#!/usr/bin/env sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p dist
VERSION="${VERSION:-dev}"
COMMIT="${COMMIT:-$(git rev-parse --short HEAD 2>/dev/null || echo unknown)}"
DATE="${DATE:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
EXTRA_LDFLAGS=""
GOOS="$(go env GOOS)"
if [ "$GOOS" = "darwin" ]; then
# Some macOS environments treat missing LC_UUID as fatal; external linkmode
# ensures the toolchain emits LC_UUID.
EXTRA_LDFLAGS="-linkmode=external"
fi
go build \
-trimpath \
-ldflags "-s -w ${EXTRA_LDFLAGS} -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" \
-o dist/ipgrep \
./cmd/ipgrep
echo "built dist/ipgrep"