bucket discovery engine: CT domain -> candidate names -> listing index
CMS SANs are custom domains, not bucket names, so matching SANs finds almost no buckets. This adds internal/discover: from each non-matching hostname it derives candidate names (domain label + subdomain label + ~40 suffixes) plus a built-in wordlist, and anonymously tests listing on S3, GCS and (optionally) DigitalOcean Spaces. Every 200 is indexed (keys/sizes/dates only). Bounded hard: global BUCKET_SCAN_QPS rate limit, 12h per-name dedup, random sampled domain queue, MaxPerDomain cap. Never downloads contents. Fix: GCS candidates now use the virtual-host form (<bucket>.storage.googleapis.com) so buckets no longer collide on the cloud_assets unique key; previously they all shared storage.googleapis.com and overwrote each other. Live: 30 open buckets in ~5 min (e.g. estimator-dev, image-prod, update-media, mos-images S3; several GCS buckets), 11k+ objects indexed.
This commit is contained in:
1 parent
816fa572df
commit
a899362e18
9 files changed
+615
-32
No files matched your search
@@ -41,6 +41,9 @@ type Metrics struct {
|
||||
ProbePublic atomic.Uint64
|
||||
EntriesIndexed atomic.Uint64
|
||||
|
||||
BucketCandidates atomic.Uint64
|
||||
BucketOpen atomic.Uint64
|
||||
|
||||
gauges []gauge
|
||||
}
|
||||
|
||||
@@ -86,6 +89,8 @@ func (m *Metrics) Handler() http.Handler {
|
||||
counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
|
||||
counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
|
||||
counter("entries_indexed_total", "Metadata entries indexed from readable resources", m.EntriesIndexed.Load())
|
||||
counter("bucket_candidates_total", "Candidate bucket names tested", m.BucketCandidates.Load())
|
||||
counter("bucket_open_total", "Open (listable) buckets discovered by name enumeration", m.BucketOpen.Load())
|
||||
for _, g := range m.gauges {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user