parallel backfill, persist scanned exposure, random-sample DNS queue
- scanner.run now persists exposure_state/detail/capabilities for scanned assets (previously only the index was written). - Backfill runs a bounded worker pool (16) so it verifies reachable assets without saturating the DB pool; DB_MAX_CONNS raised for the live test. - CNAME enrichment queue is now a bounded random sample instead of a FIFO. The FIFO filled with the first N hostnames seen at startup and dropped the rest, so CNAME discovery only ever saw stale hostnames; random replacement gives representative coverage at bounded memory (same fix as the prober). Verified live: - all four known-public resources scan + index (DO, AWS s3-<region>, GCS, Azure container path), 38-1000 objects each. - backfill of every CT-discovered 'reachable' asset (725) and 'auth_required' (12) completed with 0 listable: those are MRAP/non-bucket 400s and list-denied buckets, confirming the checks are actually running.
This commit is contained in:
1 parent
62a7d2eb2e
commit
171c9b37eb
6 files changed
+196
-57
No files matched your search
@@ -50,7 +50,7 @@ type Pipeline struct {
|
||||
cfg Config
|
||||
|
||||
directQ chan model.Candidate
|
||||
dnsQ chan string
|
||||
dnsQ *stringSample
|
||||
dbQ chan Finding
|
||||
|
||||
dedup *dns.TTLCache[bool] // hostnames already queued for DNS
|
||||
@@ -90,14 +90,14 @@ func New(cfg Config) *Pipeline {
|
||||
p := &Pipeline{
|
||||
cfg: cfg,
|
||||
directQ: make(chan model.Candidate, cfg.DirectQueueSize),
|
||||
dnsQ: make(chan string, cfg.DNSQueueSize),
|
||||
dnsQ: newStringSample(cfg.DNSQueueSize),
|
||||
dbQ: make(chan Finding, cfg.DBQueueSize),
|
||||
dedup: dns.NewTTLCache[bool](cfg.DedupSize, 64),
|
||||
seen: dns.NewTTLCache[bool](cfg.DedupSize, 64),
|
||||
}
|
||||
if p.cfg.Metrics != nil {
|
||||
p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) })
|
||||
p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(len(p.dnsQ)) })
|
||||
p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(p.dnsQ.len()) })
|
||||
p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) })
|
||||
}
|
||||
return p
|
||||
@@ -119,14 +119,14 @@ func (p *Pipeline) Start(ctx context.Context) {
|
||||
|
||||
// Depths reports the current length of the direct, DNS and DB queues.
|
||||
func (p *Pipeline) Depths() (direct, dns, db int) {
|
||||
return len(p.directQ), len(p.dnsQ), len(p.dbQ)
|
||||
return len(p.directQ), p.dnsQ.len(), len(p.dbQ)
|
||||
}
|
||||
|
||||
// Close stops intake and drains all queues. It must only be called after the
|
||||
// CertSpotter monitor has stopped calling HandleIdentifiers.
|
||||
func (p *Pipeline) Close() {
|
||||
close(p.directQ)
|
||||
close(p.dnsQ)
|
||||
p.dnsQ.close()
|
||||
p.producer.Wait()
|
||||
close(p.dbQ)
|
||||
p.workers.Wait()
|
||||
@@ -175,11 +175,7 @@ func (p *Pipeline) HandleHost(host string) {
|
||||
return
|
||||
}
|
||||
p.dedup.Set(host, true, p.cfg.DedupTTL)
|
||||
select {
|
||||
case p.dnsQ <- host:
|
||||
default:
|
||||
p.drop()
|
||||
}
|
||||
p.dnsQ.push(host)
|
||||
}
|
||||
|
||||
func (p *Pipeline) drop() {
|
||||
@@ -239,12 +235,17 @@ func (p *Pipeline) dnsWorker(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case host, ok := <-p.dnsQ:
|
||||
case <-p.dnsQ.stop:
|
||||
return
|
||||
case <-p.dnsQ.notify:
|
||||
}
|
||||
for {
|
||||
host, ok := p.dnsQ.pop()
|
||||
if !ok {
|
||||
return
|
||||
break
|
||||
}
|
||||
if len(p.dbQ) >= cap(p.dbQ)/2 {
|
||||
// Load shedding: optional CNAME enrichment is dropped before
|
||||
// Load shedding: optional CNAME enrichment is skipped before
|
||||
// direct matches when the database writer is backed up.
|
||||
p.drop()
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user