parallel backfill, persist scanned exposure, random-sample DNS queue

- scanner.run now persists exposure_state/detail/capabilities for scanned
  assets (previously only the index was written).
- Backfill runs a bounded worker pool (16) so it verifies reachable assets
  without saturating the DB pool; DB_MAX_CONNS raised for the live test.
- CNAME enrichment queue is now a bounded random sample instead of a FIFO.
  The FIFO filled with the first N hostnames seen at startup and dropped the
  rest, so CNAME discovery only ever saw stale hostnames; random replacement
  gives representative coverage at bounded memory (same fix as the prober).

Verified live:
- all four known-public resources scan + index (DO, AWS s3-<region>, GCS,
  Azure container path), 38-1000 objects each.
- backfill of every CT-discovered 'reachable' asset (725) and 'auth_required'
  (12) completed with 0 listable: those are MRAP/non-bucket 400s and
  list-denied buckets, confirming the checks are actually running.

internal/pipeline/dnsqueue.go

@@ -0,0 +1,71 @@
+package pipeline
+
+import (
+	mathrand "math/rand/v2"
+	"sync"
+)
+
+// stringSample is a memory-bounded queue that keeps a random sample of recent
+// hostnames instead of a FIFO. A FIFO fills with the first N hostnames seen at
+// startup and drops everything after, so DNS enrichment only ever sees stale
+// hostnames and CNAME discovery stalls. Random replacement makes coverage
+// representative at bounded memory. Not on the certificate hot path.
+type stringSample struct {
+	mu     sync.Mutex
+	items  []string
+	cap    int
+	notify chan struct{}
+	stop   chan struct{}
+	once   sync.Once
+}
+
+func newStringSample(capacity int) *stringSample {
+	return &stringSample{
+		cap:    capacity,
+		notify: make(chan struct{}, 1),
+		stop:   make(chan struct{}),
+	}
+}
+
+func (q *stringSample) push(s string) {
+	q.mu.Lock()
+	if len(q.items) < q.cap {
+		q.items = append(q.items, s)
+	} else {
+		q.items[mathrand.IntN(len(q.items))] = s
+	}
+	q.mu.Unlock()
+	select {
+	case q.notify <- struct{}{}:
+	default:
+	}
+}
+
+func (q *stringSample) pop() (string, bool) {
+	q.mu.Lock()
+	n := len(q.items)
+	if n == 0 {
+		q.mu.Unlock()
+		return "", false
+	}
+	i := mathrand.IntN(n)
+	s := q.items[i]
+	q.items[i] = q.items[n-1]
+	q.items = q.items[:n-1]
+	q.mu.Unlock()
+	return s, true
+}
+
+func (q *stringSample) len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return len(q.items)
+}
+
+func (q *stringSample) close() {
+	q.once.Do(func() { close(q.stop) })
+	select {
+	case q.notify <- struct{}{}:
+	default:
+	}
+}

internal/pipeline/pipeline.go

@@ -50,7 +50,7 @@ type Pipeline struct {
 	cfg Config
 
 	directQ chan model.Candidate
-	dnsQ    chan string
+	dnsQ    *stringSample
 	dbQ     chan Finding
 
 	dedup    *dns.TTLCache[bool] // hostnames already queued for DNS
@@ -90,14 +90,14 @@ func New(cfg Config) *Pipeline {
 	p := &Pipeline{
 		cfg:     cfg,
 		directQ: make(chan model.Candidate, cfg.DirectQueueSize),
-		dnsQ:    make(chan string, cfg.DNSQueueSize),
+		dnsQ:    newStringSample(cfg.DNSQueueSize),
 		dbQ:     make(chan Finding, cfg.DBQueueSize),
 		dedup:   dns.NewTTLCache[bool](cfg.DedupSize, 64),
 		seen:    dns.NewTTLCache[bool](cfg.DedupSize, 64),
 	}
 	if p.cfg.Metrics != nil {
 		p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) })
-		p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(len(p.dnsQ)) })
+		p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(p.dnsQ.len()) })
 		p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) })
 	}
 	return p
@@ -119,14 +119,14 @@ func (p *Pipeline) Start(ctx context.Context) {
 
 // Depths reports the current length of the direct, DNS and DB queues.
 func (p *Pipeline) Depths() (direct, dns, db int) {
-	return len(p.directQ), len(p.dnsQ), len(p.dbQ)
+	return len(p.directQ), p.dnsQ.len(), len(p.dbQ)
 }
 
 // Close stops intake and drains all queues. It must only be called after the
 // CertSpotter monitor has stopped calling HandleIdentifiers.
 func (p *Pipeline) Close() {
 	close(p.directQ)
-	close(p.dnsQ)
+	p.dnsQ.close()
 	p.producer.Wait()
 	close(p.dbQ)
 	p.workers.Wait()
@@ -175,11 +175,7 @@ func (p *Pipeline) HandleHost(host string) {
 		return
 	}
 	p.dedup.Set(host, true, p.cfg.DedupTTL)
-	select {
-	case p.dnsQ <- host:
-	default:
-		p.drop()
-	}
+	p.dnsQ.push(host)
 }
 
 func (p *Pipeline) drop() {
@@ -239,12 +235,17 @@ func (p *Pipeline) dnsWorker(ctx context.Context) {
 		select {
 		case <-ctx.Done():
 			return
-		case host, ok := <-p.dnsQ:
+		case <-p.dnsQ.stop:
+			return
+		case <-p.dnsQ.notify:
+		}
+		for {
+			host, ok := p.dnsQ.pop()
 			if !ok {
-				return
+				break
 			}
 			if len(p.dbQ) >= cap(p.dbQ)/2 {
-				// Load shedding: optional CNAME enrichment is dropped before
+				// Load shedding: optional CNAME enrichment is skipped before
 				// direct matches when the database writer is backed up.
 				p.drop()
 				continue

internal/pipeline/pipeline_test.go

@@ -269,7 +269,7 @@ func TestDNSDedup(t *testing.T) {
 	p := New(Config{DNSQueueSize: 8, Metrics: m})
 	p.HandleHost("files.example.com")
 	p.HandleHost("files.example.com")
-	if got := len(p.dnsQ); got != 1 {
+	if got := p.dnsQ.len(); got != 1 {
 		t.Fatalf("dns queue len = %d, want 1", got)
 	}
 }