Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs

Logging:
- structured slog with LOG_FORMAT=json|text and a stable key vocabulary
- periodic stats record with queue depths, db pool, goroutines, memory, dns/probe
- no per-certificate logging (hot path is counters only)

UI (Bucketeer branding, DESIGN.md):
- light+dark themes (OS default, persisted header toggle, no first-paint flash)
- official mascot: lossless WebP light/dark logos + favicon, embedded and served
  with cache headers (1.2MB source PNGs -> ~27/22KB webp)
- Assets/Contents views, pagination (offset) and native virtual scrolling
  (content-visibility) so large result sets do not blow up the DOM
- search with debounce/abort + URL state, exposure chips, sort, summary counts

Readability + indexing:
- internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch
  (metadata only: keys/sizes/dates; never object contents)
- asset_entries table + store write/search; readable resources are indexed and
  searchable in the UI
- deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING,
  PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune
  HTTP reads
- Azure account-level listing fixed: 403 is reported reachable, not
  auth_required, since anonymous access is per-container

DNS:
- curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in
  Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at
  once broke the container resolver)
- random healthy selection + failover + per-server cooldown

Other: prober Close fixes a shutdown hang; config/README updated.

internal/dns/pool.go

@@ -2,58 +2,181 @@ package dns
 
 import (
 	"context"
+	_ "embed"
 	"errors"
+	mathrand "math/rand/v2"
 	"net"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 )
 
-// ServerPool load-balances DNS lookups across multiple resolvers using
-// round-robin, and fails over to the next server on a timeout or server error.
-// Definitive answers (NXDOMAIN) are returned immediately and are not retried on
-// every server.
-// defaultAttemptTimeout bounds each individual server attempt so a dead
-// resolver costs a couple of seconds before failover, not the entire lookup
-// budget.
-const defaultAttemptTimeout = 2 * time.Second
+//go:embed resolvers.txt
+var embeddedResolvers string
 
+// EmbeddedServers returns the compiled-in public resolver list (a snapshot of
+// proabiral/Fresh-Resolvers). Refresh it with:
+//
+//	curl -sL https://raw.githubusercontent.com/proabiral/Fresh-Resolvers/master/resolvers.txt -o internal/dns/resolvers.txt
+//
+// These are untrusted third-party resolvers: they can observe and, in theory,
+// forge DNS answers. Use DNS_SERVERS with your own resolvers for sensitive
+// deployments. A circuit breaker skips ones that stop responding.
+func EmbeddedServers() []string {
+	lines := strings.Split(embeddedResolvers, "\n")
+	out := make([]string, 0, len(lines))
+	for _, l := range lines {
+		if l = strings.TrimSpace(l); l != "" {
+			out = append(out, l)
+		}
+	}
+	return out
+}
+
+// CuratedServers are well-known, reliable public resolvers. They are always
+// included (ahead of the community list) so a mostly-dead list still works.
+func CuratedServers() []string {
+	return []string{
+		"1.1.1.1", "1.0.0.1", // Cloudflare
+		"8.8.8.8", "8.8.4.4", // Google
+		"9.9.9.9", "149.112.112.112", // Quad9
+		"208.67.222.222", "208.67.220.220", // OpenDNS
+		"94.140.14.14", "94.140.15.15", // AdGuard
+		"76.76.2.0", "76.76.10.0", // ControlD
+		"185.228.168.9", "185.228.169.9", // CleanBrowsing
+		"156.154.70.1", "156.154.71.1", // Neustar
+		"64.6.64.6", "64.6.65.6", // Verisign
+		"74.82.42.42",                    // Hurricane Electric
+		"216.146.35.35", "216.146.36.36", // Oracle/Dyn
+	}
+}
+
+const (
+	defaultAttemptTimeout  = 2 * time.Second
+	maxQueryAttempts       = 3
+	serverFailureThreshold = 3
+	serverCooldown         = 5 * time.Minute
+)
+
+type server struct {
+	r         Resolver
+	failures  atomic.Int32
+	downUntil atomic.Int64 // unix nanoseconds; 0 = healthy
+}
+
+func (s *server) available(now int64) bool { return s.downUntil.Load() <= now }
+
+func (s *server) success() {
+	s.failures.Store(0)
+	s.downUntil.Store(0)
+}
+
+func (s *server) failure(now int64) {
+	if s.failures.Add(1) >= serverFailureThreshold {
+		s.downUntil.Store(now + int64(serverCooldown))
+		s.failures.Store(0)
+	}
+}
+
+// ServerPool load-balances DNS lookups across many resolvers. Each query picks
+// a random healthy resolver and, on a transient error, fails over to a bounded
+// number of others. Resolvers that repeatedly fail are put in cooldown, so a
+// large list of mostly-dead public resolvers converges on the working subset.
+//
+// ponytail: random selection + cooldown, not latency-weighted or health-scored.
+// Upgrade to EWMA latency weighting if query latency matters.
 type ServerPool struct {
-	resolvers      []Resolver
-	next           atomic.Uint64
+	servers        []*server
 	attemptTimeout time.Duration
 }
 
-func (p *ServerPool) withAttemptTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
-	t := p.attemptTimeout
-	if t <= 0 {
-		t = defaultAttemptTimeout
-	}
-	return context.WithTimeout(ctx, t)
-}
-
-// NewServerPool builds a pool from "host", "host:port" or "system" entries.
-// An empty list uses the system resolver.
+// NewServerPool builds a pool. Entries may be "host", "host:port" or "system".
+// An empty list falls back to the OS resolver.
 func NewServerPool(servers []string) *ServerPool {
-	var resolvers []Resolver
+	p := &ServerPool{}
 	for _, s := range servers {
 		s = strings.TrimSpace(s)
 		switch {
 		case s == "":
 			continue
 		case s == "system" || s == "local":
-			return &ServerPool{resolvers: []Resolver{&GoResolver{R: net.DefaultResolver}}}
+			p.servers = append(p.servers, &server{r: &GoResolver{R: net.DefaultResolver}})
 		default:
 			if !strings.Contains(s, ":") {
 				s += ":53"
 			}
-			resolvers = append(resolvers, &GoResolver{R: newServerResolver(s)})
+			p.servers = append(p.servers, &server{r: &GoResolver{R: newServerResolver(s)}})
 		}
 	}
-	if len(resolvers) == 0 {
-		resolvers = append(resolvers, &GoResolver{R: net.DefaultResolver})
+	if len(p.servers) == 0 {
+		p.servers = append(p.servers, &server{r: &GoResolver{R: net.DefaultResolver}})
 	}
-	return &ServerPool{resolvers: resolvers}
+	return p
+}
+
+// Len reports how many resolvers are in the pool.
+func (p *ServerPool) Len() int { return len(p.servers) }
+
+// Warm probes up to maxProbe randomly chosen resolvers and puts non-responders
+// into cooldown, so a large, partly-stale public list converges on a working
+// subset. It deliberately samples instead of probing the whole list: firing
+// thousands of concurrent DNS queries can overwhelm the host/container
+// resolver and break unrelated lookups. maxProbe <= 0 means all. Returns the
+// healthy count.
+func (p *ServerPool) Warm(ctx context.Context, probeHost string, workers, maxProbe int) int {
+	if workers <= 0 {
+		workers = 32
+	}
+	servers := p.servers
+	if maxProbe > 0 && maxProbe < len(servers) {
+		order := make([]int, len(servers))
+		for i := range order {
+			order[i] = i
+		}
+		mathrand.Shuffle(len(order), func(i, j int) { order[i], order[j] = order[j], order[i] })
+		servers = make([]*server, maxProbe)
+		for i := range servers {
+			servers[i] = p.servers[order[i]]
+		}
+	}
+	probeTimeout := time.Second
+	if p.attemptTimeout > 0 && p.attemptTimeout < probeTimeout {
+		probeTimeout = p.attemptTimeout
+	}
+	jobs := make(chan *server)
+	var (
+		wg      sync.WaitGroup
+		healthy atomic.Int32
+	)
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for s := range jobs {
+				rctx, cancel := context.WithTimeout(ctx, probeTimeout)
+				_, err := s.r.LookupIP(rctx, probeHost)
+				cancel()
+				if err == nil || definitive(err) {
+					healthy.Add(1) // answered, even if NXDOMAIN
+					continue
+				}
+				s.downUntil.Store(time.Now().Add(serverCooldown).UnixNano())
+			}
+		}()
+	}
+	for _, s := range servers {
+		select {
+		case jobs <- s:
+		case <-ctx.Done():
+			close(jobs)
+			wg.Wait()
+			return int(healthy.Load())
+		}
+	}
+	close(jobs)
+	wg.Wait()
+	return int(healthy.Load())
 }
 
 func newServerResolver(server string) *net.Resolver {
@@ -69,8 +192,48 @@ func newServerResolver(server string) *net.Resolver {
 	}
 }
 
-func (p *ServerPool) pick() int {
-	return int((p.next.Add(1) - 1) % uint64(len(p.resolvers)))
+func (p *ServerPool) withAttemptTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
+	t := p.attemptTimeout
+	if t <= 0 {
+		t = defaultAttemptTimeout
+	}
+	return context.WithTimeout(ctx, t)
+}
+
+func triedContain(tried []int, n, idx int) bool {
+	for i := 0; i < n; i++ {
+		if tried[i] == idx {
+			return true
+		}
+	}
+	return false
+}
+
+// pickIndex returns an untried resolver, preferring healthy ones, by scanning
+// cyclically from a random start. Scanning (rather than random sampling) means
+// a healthy resolver is always found when one exists, which matters for small
+// pools. With a warm pool of thousands the first candidate is almost always
+// healthy, so it stays O(1) in practice. If all are in cooldown it still
+// returns one, so a fully-cooled pool can recover.
+func (p *ServerPool) pickIndex(now int64, tried []int, n int) int {
+	total := len(p.servers)
+	if total == 0 {
+		return -1
+	}
+	start := mathrand.IntN(total)
+	for i := 0; i < total; i++ {
+		idx := (start + i) % total
+		if !triedContain(tried, n, idx) && p.servers[idx].available(now) {
+			return idx
+		}
+	}
+	for i := 0; i < total; i++ {
+		idx := (start + i) % total
+		if !triedContain(tried, n, idx) {
+			return idx
+		}
+	}
+	return -1
 }
 
 // definitive reports whether err is a real DNS answer (NXDOMAIN) that should not
@@ -80,40 +243,56 @@ func definitive(err error) bool {
 	return errors.As(err, &dnsErr) && dnsErr.IsNotFound
 }
 
-func (p *ServerPool) LookupCNAME(ctx context.Context, host string) (string, error) {
-	n := len(p.resolvers)
-	start := p.pick()
-	var last error
-	for i := 0; i < n; i++ {
+func (p *ServerPool) lookup(ctx context.Context, fn func(context.Context, Resolver) error) error {
+	var (
+		tried [maxQueryAttempts]int
+		last  error
+	)
+	for n := 0; n < maxQueryAttempts; n++ {
+		idx := p.pickIndex(time.Now().UnixNano(), tried[:], n)
+		if idx < 0 {
+			break
+		}
+		tried[n] = idx
 		actx, cancel := p.withAttemptTimeout(ctx)
-		cname, err := p.resolvers[(start+i)%n].LookupCNAME(actx, host)
+		err := fn(actx, p.servers[idx].r)
 		cancel()
 		if err == nil {
-			return cname, nil
+			p.servers[idx].success()
+			return nil
 		}
 		if definitive(err) {
-			return "", err
+			return err
 		}
+		p.servers[idx].failure(time.Now().UnixNano())
 		last = err
+		if ctx.Err() != nil {
+			return err
+		}
 	}
-	return "", last
+	return last
+}
+
+func (p *ServerPool) LookupCNAME(ctx context.Context, host string) (string, error) {
+	var result string
+	err := p.lookup(ctx, func(c context.Context, r Resolver) error {
+		v, e := r.LookupCNAME(c, host)
+		if e == nil {
+			result = v
+		}
+		return e
+	})
+	return result, err
 }
 
 func (p *ServerPool) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
-	n := len(p.resolvers)
-	start := p.pick()
-	var last error
-	for i := 0; i < n; i++ {
-		actx, cancel := p.withAttemptTimeout(ctx)
-		ips, err := p.resolvers[(start+i)%n].LookupIP(actx, host)
-		cancel()
-		if err == nil {
-			return ips, nil
+	var result []net.IP
+	err := p.lookup(ctx, func(c context.Context, r Resolver) error {
+		v, e := r.LookupIP(c, host)
+		if e == nil {
+			result = v
 		}
-		if definitive(err) {
-			return nil, err
-		}
-		last = err
-	}
-	return nil, last
+		return e
+	})
+	return result, err
 }

internal/dns/pool_test.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"sync/atomic"
 	"testing"
+	"time"
 )
 
 type fakeLookup struct {
@@ -29,40 +30,67 @@ func (f *fakeLookup) LookupIP(ctx context.Context, host string) ([]net.IP, error
 	return []net.IP{net.ParseIP("1.2.3.4")}, nil
 }
 
-func TestServerPoolRoundRobin(t *testing.T) {
-	a, b := &fakeLookup{}, &fakeLookup{}
-	p := &ServerPool{resolvers: []Resolver{a, b}}
-	for i := 0; i < 4; i++ {
-		if _, err := p.LookupIP(context.Background(), "example.com"); err != nil {
-			t.Fatal(err)
+func TestEmbeddedServers(t *testing.T) {
+	if n := len(EmbeddedServers()); n < 1000 {
+		t.Fatalf("embedded resolver list has %d entries, want thousands", n)
+	}
+	if n := len(CuratedServers()); n < 10 {
+		t.Fatalf("curated resolver list has %d entries, want at least 10", n)
+	}
+}
+
+func TestServerCooldown(t *testing.T) {
+	s := &server{}
+	now := time.Now().UnixNano()
+	for i := 0; i < serverFailureThreshold; i++ {
+		s.failure(now)
+	}
+	if s.available(now) {
+		t.Fatal("server should be in cooldown after repeated failures")
+	}
+	if !s.available(now + int64(serverCooldown) + 1) {
+		t.Fatal("server should be available after cooldown")
+	}
+	s.success()
+	if !s.available(now) || s.failures.Load() != 0 {
+		t.Fatal("success should clear failures and cooldown")
+	}
+}
+
+func TestPoolFailover(t *testing.T) {
+	bad := &fakeLookup{ipErr: &net.DNSError{Err: "server misbehaving", IsTemporary: true}}
+	good := &fakeLookup{}
+	p := &ServerPool{servers: []*server{{r: bad}, {r: good}}}
+	for i := 0; i < 50; i++ {
+		ips, err := p.LookupIP(context.Background(), "example.com")
+		if err != nil || len(ips) != 1 {
+			t.Fatalf("lookup %d failed: %v", i, err)
 		}
 	}
-	if a.calls.Load() != 2 || b.calls.Load() != 2 {
-		t.Fatalf("round robin calls: a=%d b=%d, want 2/2", a.calls.Load(), b.calls.Load())
-	}
 }
 
-func TestServerPoolFailover(t *testing.T) {
-	a := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}}
-	b := &fakeLookup{}
-	p := &ServerPool{resolvers: []Resolver{a, b}}
-	ips, err := p.LookupIP(context.Background(), "example.com")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(ips) != 1 || a.calls.Load() != 1 {
-		t.Fatalf("expected failover to second server, a.calls=%d", a.calls.Load())
-	}
-}
-
-func TestServerPoolNoFailoverOnNXDOMAIN(t *testing.T) {
+func TestPoolNoFailoverOnNXDOMAIN(t *testing.T) {
 	a := &fakeLookup{ipErr: &net.DNSError{Err: "no such host", IsNotFound: true}}
-	b := &fakeLookup{}
-	p := &ServerPool{resolvers: []Resolver{a, b}}
+	b := &fakeLookup{ipErr: &net.DNSError{Err: "no such host", IsNotFound: true}}
+	p := &ServerPool{servers: []*server{{r: a}, {r: b}}}
 	if _, err := p.LookupIP(context.Background(), "missing.example.com"); err == nil {
 		t.Fatal("expected error")
 	}
-	if b.calls.Load() != 0 {
-		t.Fatalf("NXDOMAIN must not fail over; b.calls=%d", b.calls.Load())
+	if got := a.calls.Load() + b.calls.Load(); got != 1 {
+		t.Fatalf("NXDOMAIN made %d lookups, want exactly 1 (no failover)", got)
+	}
+}
+
+func TestWarmDisablesDeadResolvers(t *testing.T) {
+	good := &fakeLookup{}
+	dead := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}}
+	deadServer := &server{r: dead}
+	p := &ServerPool{servers: []*server{deadServer, {r: good}}}
+	healthy := p.Warm(context.Background(), "example.com", 4, 0)
+	if healthy != 1 {
+		t.Fatalf("healthy = %d, want 1", healthy)
+	}
+	if deadServer.available(time.Now().UnixNano()) {
+		t.Fatal("dead resolver should be in cooldown after warmup")
 	}
 }