StasonJatham
171c9b37eb
- scanner.run now persists exposure_state/detail/capabilities for scanned assets (previously only the index was written). - Backfill runs a bounded worker pool (16) so it verifies reachable assets without saturating the DB pool; DB_MAX_CONNS raised for the live test. - CNAME enrichment queue is now a bounded random sample instead of a FIFO. The FIFO filled with the first N hostnames seen at startup and dropped the rest, so CNAME discovery only ever saw stale hostnames; random replacement gives representative coverage at bounded memory (same fix as the prober). Verified live: - all four known-public resources scan + index (DO, AWS s3-<region>, GCS, Azure container path), 38-1000 objects each. - backfill of every CT-discovered 'reachable' asset (725) and 'auth_required' (12) completed with 0 listable: those are MRAP/non-bucket 400s and list-denied buckets, confirming the checks are actually running.
72 lines
1.4 KiB
Go
72 lines
1.4 KiB
Go
package pipeline
|
|
|
|
import (
|
|
mathrand "math/rand/v2"
|
|
"sync"
|
|
)
|
|
|
|
// stringSample is a memory-bounded queue that keeps a random sample of recent
|
|
// hostnames instead of a FIFO. A FIFO fills with the first N hostnames seen at
|
|
// startup and drops everything after, so DNS enrichment only ever sees stale
|
|
// hostnames and CNAME discovery stalls. Random replacement makes coverage
|
|
// representative at bounded memory. Not on the certificate hot path.
|
|
type stringSample struct {
|
|
mu sync.Mutex
|
|
items []string
|
|
cap int
|
|
notify chan struct{}
|
|
stop chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func newStringSample(capacity int) *stringSample {
|
|
return &stringSample{
|
|
cap: capacity,
|
|
notify: make(chan struct{}, 1),
|
|
stop: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
func (q *stringSample) push(s string) {
|
|
q.mu.Lock()
|
|
if len(q.items) < q.cap {
|
|
q.items = append(q.items, s)
|
|
} else {
|
|
q.items[mathrand.IntN(len(q.items))] = s
|
|
}
|
|
q.mu.Unlock()
|
|
select {
|
|
case q.notify <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (q *stringSample) pop() (string, bool) {
|
|
q.mu.Lock()
|
|
n := len(q.items)
|
|
if n == 0 {
|
|
q.mu.Unlock()
|
|
return "", false
|
|
}
|
|
i := mathrand.IntN(n)
|
|
s := q.items[i]
|
|
q.items[i] = q.items[n-1]
|
|
q.items = q.items[:n-1]
|
|
q.mu.Unlock()
|
|
return s, true
|
|
}
|
|
|
|
func (q *stringSample) len() int {
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
return len(q.items)
|
|
}
|
|
|
|
func (q *stringSample) close() {
|
|
q.once.Do(func() { close(q.stop) })
|
|
select {
|
|
case q.notify <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|