parallel backfill, persist scanned exposure, random-sample DNS queue

- scanner.run now persists exposure_state/detail/capabilities for scanned
  assets (previously only the index was written).
- Backfill runs a bounded worker pool (16) so it verifies reachable assets
  without saturating the DB pool; DB_MAX_CONNS raised for the live test.
- CNAME enrichment queue is now a bounded random sample instead of a FIFO.
  The FIFO filled with the first N hostnames seen at startup and dropped the
  rest, so CNAME discovery only ever saw stale hostnames; random replacement
  gives representative coverage at bounded memory (same fix as the prober).

Verified live:
- all four known-public resources scan + index (DO, AWS s3-<region>, GCS,
  Azure container path), 38-1000 objects each.
- backfill of every CT-discovered 'reachable' asset (725) and 'auth_required'
  (12) completed with 0 listable: those are MRAP/non-bucket 400s and
  list-denied buckets, confirming the checks are actually running.
This commit is contained in:
StasonJatham committed 2026-09-22 15:09:34 +02:00
1 parent 62a7d2eb2e
commit 171c9b37eb
6 files changed
+196 -57

No files matched your search

+14 -13
View File
@@ -50,7 +50,7 @@ type Pipeline struct {
cfg Config
directQ chan model.Candidate
dnsQ chan string
dnsQ *stringSample
dbQ chan Finding
dedup *dns.TTLCache[bool] // hostnames already queued for DNS
@@ -90,14 +90,14 @@ func New(cfg Config) *Pipeline {
p := &Pipeline{
cfg: cfg,
directQ: make(chan model.Candidate, cfg.DirectQueueSize),
dnsQ: make(chan string, cfg.DNSQueueSize),
dnsQ: newStringSample(cfg.DNSQueueSize),
dbQ: make(chan Finding, cfg.DBQueueSize),
dedup: dns.NewTTLCache[bool](cfg.DedupSize, 64),
seen: dns.NewTTLCache[bool](cfg.DedupSize, 64),
}
if p.cfg.Metrics != nil {
p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) })
p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(len(p.dnsQ)) })
p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(p.dnsQ.len()) })
p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) })
}
return p
@@ -119,14 +119,14 @@ func (p *Pipeline) Start(ctx context.Context) {
// Depths reports the current length of the direct, DNS and DB queues.
func (p *Pipeline) Depths() (direct, dns, db int) {
return len(p.directQ), len(p.dnsQ), len(p.dbQ)
return len(p.directQ), p.dnsQ.len(), len(p.dbQ)
}
// Close stops intake and drains all queues. It must only be called after the
// CertSpotter monitor has stopped calling HandleIdentifiers.
func (p *Pipeline) Close() {
close(p.directQ)
close(p.dnsQ)
p.dnsQ.close()
p.producer.Wait()
close(p.dbQ)
p.workers.Wait()
@@ -175,11 +175,7 @@ func (p *Pipeline) HandleHost(host string) {
return
}
p.dedup.Set(host, true, p.cfg.DedupTTL)
select {
case p.dnsQ <- host:
default:
p.drop()
}
p.dnsQ.push(host)
}
func (p *Pipeline) drop() {
@@ -239,12 +235,17 @@ func (p *Pipeline) dnsWorker(ctx context.Context) {
select {
case <-ctx.Done():
return
case host, ok := <-p.dnsQ:
case <-p.dnsQ.stop:
return
case <-p.dnsQ.notify:
}
for {
host, ok := p.dnsQ.pop()
if !ok {
return
break
}
if len(p.dbQ) >= cap(p.dbQ)/2 {
// Load shedding: optional CNAME enrichment is dropped before
// Load shedding: optional CNAME enrichment is skipped before
// direct matches when the database writer is backed up.
p.drop()
continue