From a899362e18e7e0fa81d37a05e955353f5033f844 Mon Sep 17 00:00:00 2001 From: StasonJatham Date: Tue, 22 Sep 2026 15:53:31 +0200 Subject: [PATCH] bucket discovery engine: CT domain -> candidate names -> listing index CMS SANs are custom domains, not bucket names, so matching SANs finds almost no buckets. This adds internal/discover: from each non-matching hostname it derives candidate names (domain label + subdomain label + ~40 suffixes) plus a built-in wordlist, and anonymously tests listing on S3, GCS and (optionally) DigitalOcean Spaces. Every 200 is indexed (keys/sizes/dates only). Bounded hard: global BUCKET_SCAN_QPS rate limit, 12h per-name dedup, random sampled domain queue, MaxPerDomain cap. Never downloads contents. Fix: GCS candidates now use the virtual-host form (.storage.googleapis.com) so buckets no longer collide on the cloud_assets unique key; previously they all shared storage.googleapis.com and overwrote each other. Live: 30 open buckets in ~5 min (e.g. estimator-dev, image-prod, update-media, mos-images S3; several GCS buckets), 11k+ objects indexed. --- .env.example | 7 + README.md | 29 ++ cmd/ctcloud/main.go | 43 +++ internal/config/config.go | 75 +++--- internal/discover/discover.go | 415 +++++++++++++++++++++++++++++ internal/discover/discover_test.go | 63 +++++ internal/metrics/metrics.go | 5 + internal/model/model.go | 3 + internal/pipeline/pipeline.go | 7 + 9 files changed, 615 insertions(+), 32 deletions(-) create mode 100644 internal/discover/discover.go create mode 100644 internal/discover/discover_test.go diff --git a/.env.example b/.env.example index 5b293e4..0822c33 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,10 @@ INDEX_MAX_OBJECTS=1000 # Optional HTTP basic auth for the web UI (/ and /api/*): # UI_USER=admin # UI_PASS=change-me-strong + +# Bucket-name discovery (finds open buckets; CT SANs are domains, not buckets) +BUCKET_SCAN_ENABLED=true +BUCKET_SCAN_QPS=50 +BUCKET_SCAN_WORKERS=16 +BUCKET_SCAN_QUEUE=65536 +BUCKET_SCAN_DO=false diff --git a/README.md b/README.md index 14b014c..c4b36c6 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,11 @@ Environment variables only; no config files, no runtime pattern files. | `PROBE_DEEP` | `true` | unauthenticated, read-only protocol reads for databases (Redis/Postgres/Mongo) | | `INDEX_ENABLED` / `INDEX_MAX_OBJECTS` | `true` / `1000` | index metadata of readable resources (keys/sizes only) | | `LOG_FORMAT` | `text` | `text` or `json` (JSON is best for log pipelines/LLM analysis) | +| `BUCKET_SCAN_ENABLED` | `true` | discover open buckets by name enumeration | +| `BUCKET_SCAN_QPS` | `50` | global cap on outbound discovery requests | +| `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers | +| `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog | +| `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions | | `DEDUP_TTL` | `1h` | hostname/finding dedup window | | `METRICS_ADDR` | `:9090` | metrics/health/UI listener | | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI | @@ -305,6 +310,30 @@ go test ./internal/classify/ ./internal/pipeline/ -run '^$' -bench . -benchmem `BenchmarkClassifyMixed` and `TestHandleHostNoAllocs` fail unless the result is exactly 0 allocs/op, so the target is enforced by the test suite. +### Bucket discovery + +Certificate SANs are custom domains, not bucket names, so matching SANs alone +finds almost no buckets. Instead, every non-matching hostname feeds a discovery +engine that: + +1. derives candidate names from the domain (`example` → `example`, + `example-assets`, `example-backups`, …, plus the subdomain label), also + testing a built-in wordlist of common names; +2. tests anonymous listing per provider — + `https://.s3.amazonaws.com/?list-type=2` (AWS), + `https://.storage.googleapis.com` (GCS), + `https://..digitaloceanspaces.com` (DO, optional); +3. indexes every bucket that returns 200 (keys/sizes/dates only). + +It is aggressively bounded: a global `BUCKET_SCAN_QPS` limit, a 12h per-name +dedup cache, and a random-sampled domain queue. Azure cannot be enumerated this +way (account names are the namespace and containers need names), so Azure is +handled from CT accounts plus a container wordlist, and R2 is never listable. + +This is what actually yields open buckets. Of the buckets found, S3 and GCS +return full object listings; Azure needs a known container; R2 only serves +objects you already know. + ### DNS load balancing When `DNS_SERVERS` is unset, the pool is ~20 curated, reliable public resolvers diff --git a/cmd/ctcloud/main.go b/cmd/ctcloud/main.go index c156cd7..01112a2 100644 --- a/cmd/ctcloud/main.go +++ b/cmd/ctcloud/main.go @@ -20,6 +20,7 @@ import ( "software.sslmate.com/src/certspotter/monitor" "tea.karlcom.de/karl/bucketeer/internal/config" + "tea.karlcom.de/karl/bucketeer/internal/discover" "tea.karlcom.de/karl/bucketeer/internal/dns" "tea.karlcom.de/karl/bucketeer/internal/index" "tea.karlcom.de/karl/bucketeer/internal/metrics" @@ -145,6 +146,35 @@ func run() error { } scanner := &scan.Scanner{Prober: scanProber, Indexer: indexer, Enricher: enricher, Store: st, BackfillWorkers: backfillWorkers} + var discoverer *discover.Discoverer + if cfg.BucketScanEnabled { + discoverer = discover.New(discover.Config{ + QPS: cfg.BucketScanQPS, Workers: cfg.BucketScanWorkers, Queue: cfg.BucketScanQueue, + S3: true, GCS: true, DOSpaces: cfg.BucketScanDO, + DORegions: []string{"nyc3", "sfo3", "fra1", "ams3", "sgp1"}, + Indexer: indexer, Metrics: m, + OnOpen: func(uctx context.Context, c model.Candidate, detail string, entries []model.Entry) { + if _, err := st.EnsureAsset(uctx, c); err != nil { + return + } + _ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, + model.ExposurePublicRead, detail, true, true) + if n, err := st.ReplaceEntries(uctx, c.Provider.String(), c.Service.String(), c.Hostname, entries); err == nil && n > 0 { + m.EntriesIndexed.Add(uint64(n)) + } + slog.Info("open bucket discovered", "event", "open_bucket", + "host", c.Hostname, "provider", c.Provider.String(), "service", c.Service.String(), "entries", len(entries)) + }, + }) + discoverer.Start(ctx) + for _, name := range discover.Wordlist() { + discoverer.SubmitName(name) + } + slog.Info("bucket name discovery enabled", "event", "startup", + "qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO, + "wordlist", len(discover.Wordlist())) + } + var prober *probe.Prober if cfg.ActiveProbing { pcfg := probe.Config{ @@ -184,6 +214,10 @@ func run() error { return err } + var discoverField interface{ Submit(host string) } + if discoverer != nil { + discoverField = discoverer + } p := pipeline.New(pipeline.Config{ DirectQueueSize: cfg.DirectQueueSize, DNSQueueSize: cfg.DNSQueueSize, @@ -196,6 +230,7 @@ func run() error { Enricher: enricher, Write: write, Metrics: m, + Discover: discoverField, }) p.Start(ctx) @@ -237,6 +272,10 @@ func run() error { prober.Close() prober.Wait() } + if discoverer != nil { + discoverer.Close() + discoverer.Wait() + } close(drained) }() select { @@ -339,6 +378,10 @@ func logStats(ctx context.Context, m *metrics.Metrics, p *pipeline.Pipeline, st "probe_public", m.ProbePublic.Load(), ) } + attrs = append(attrs, + "bucket_candidates", m.BucketCandidates.Load(), + "bucket_open", m.BucketOpen.Load(), + ) slog.Info("periodic stats", attrs...) } } diff --git a/internal/config/config.go b/internal/config/config.go index 9fa7f98..993bc20 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,6 +39,12 @@ type Config struct { IndexEnabled bool IndexMaxObjects int + BucketScanEnabled bool + BucketScanQPS int + BucketScanWorkers int + BucketScanQueue int + BucketScanDO bool + MetricsAddr string UIUser string UIPass string @@ -57,38 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js func Load() (*Config, error) { c := &Config{ - DatabaseURL: os.Getenv("DATABASE_URL"), - DBMaxConns: envInt("DB_MAX_CONNS", 8), - DNSServers: envList("DNS_SERVERS", nil), - DNSWorkers: envInt("DNS_WORKERS", 64), - DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), - DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), - DedupSize: envInt("DEDUP_SIZE", 1<<18), - DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), - DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), - DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), - DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), - DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), - DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), - DBBatchSize: envInt("DB_BATCH_SIZE", 500), - DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), - ActiveProbing: envBool("ACTIVE_PROBING", false), - ProbeWorkers: envInt("PROBE_WORKERS", 8), - ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), - ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), - ProbeDeep: envBool("PROBE_DEEP", true), - IndexEnabled: envBool("INDEX_ENABLED", true), - IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), - MetricsAddr: envString("METRICS_ADDR", ":9090"), - UIUser: envString("UI_USER", ""), - UIPass: envString("UI_PASS", ""), - LogLevel: envString("LOG_LEVEL", "info"), - LogFormat: envString("LOG_FORMAT", "text"), - StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), - LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), - StartAtEnd: envBool("START_AT_END", false), - ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), - DedupeTTL: envDuration("DEDUP_TTL", time.Hour), + DatabaseURL: os.Getenv("DATABASE_URL"), + DBMaxConns: envInt("DB_MAX_CONNS", 8), + DNSServers: envList("DNS_SERVERS", nil), + DNSWorkers: envInt("DNS_WORKERS", 64), + DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), + DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), + DedupSize: envInt("DEDUP_SIZE", 1<<18), + DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), + DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), + DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), + DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), + DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), + DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), + DBBatchSize: envInt("DB_BATCH_SIZE", 500), + DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), + ActiveProbing: envBool("ACTIVE_PROBING", false), + ProbeWorkers: envInt("PROBE_WORKERS", 8), + ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), + ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), + ProbeDeep: envBool("PROBE_DEEP", true), + IndexEnabled: envBool("INDEX_ENABLED", true), + IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), + BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true), + BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50), + BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16), + BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536), + BucketScanDO: envBool("BUCKET_SCAN_DO", false), + MetricsAddr: envString("METRICS_ADDR", ":9090"), + UIUser: envString("UI_USER", ""), + UIPass: envString("UI_PASS", ""), + LogLevel: envString("LOG_LEVEL", "info"), + LogFormat: envString("LOG_FORMAT", "text"), + StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), + LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), + StartAtEnd: envBool("START_AT_END", false), + ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), + DedupeTTL: envDuration("DEDUP_TTL", time.Hour), } if err := c.validate(); err != nil { return nil, err diff --git a/internal/discover/discover.go b/internal/discover/discover.go new file mode 100644 index 0000000..da6caf7 --- /dev/null +++ b/internal/discover/discover.go @@ -0,0 +1,415 @@ +// Package discover turns the domains seen in CT into candidate object-storage +// bucket names and tests them anonymously. This is the engine that actually +// finds open buckets: certificate SANs are custom domains, not bucket names, so +// we generate plausible names and ask the providers' list APIs. +// +// It is aggressively bounded: a global rate limiter caps outbound requests, +// candidate names are deduplicated with a TTL cache, and the domain intake is a +// random sample so no single domain floods the queue. Contents are never +// downloaded; only the list API is called. +package discover + +import ( + "context" + "io" + "math/rand/v2" + "net/http" + "strings" + "sync" + "time" + + "tea.karlcom.de/karl/bucketeer/internal/dns" + "tea.karlcom.de/karl/bucketeer/internal/index" + "tea.karlcom.de/karl/bucketeer/internal/metrics" + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +type Config struct { + QPS int // global outbound request rate + Workers int // concurrent candidate probes + Queue int // sampled domain backlog + S3 bool + GCS bool + DOSpaces bool + DORegions []string + MaxPerDomain int // cap candidates generated per domain + + Indexer *index.Indexer + Metrics *metrics.Metrics + // OnOpen is called for every anonymously listable bucket with its metadata + // index. contents are never downloaded. + OnOpen func(ctx context.Context, c model.Candidate, detail string, entries []model.Entry) +} + +type Discoverer struct { + cfg Config + client *http.Client + q *domainQueue + names *domainQueue + tokens chan struct{} + seen *dns.TTLCache[bool] // candidate names already tested + stop chan struct{} + wg sync.WaitGroup +} + +func New(cfg Config) *Discoverer { + if cfg.QPS <= 0 { + cfg.QPS = 50 + } + if cfg.Workers <= 0 { + cfg.Workers = 16 + } + if cfg.Queue <= 0 { + cfg.Queue = 65536 + } + if cfg.MaxPerDomain <= 0 { + cfg.MaxPerDomain = 48 + } + return &Discoverer{ + cfg: cfg, + client: &http.Client{Timeout: 6 * time.Second}, + q: newDomainQueue(cfg.Queue), + names: newDomainQueue(cfg.Queue), + tokens: make(chan struct{}, max(cfg.QPS, 1)), + seen: dns.NewTTLCache[bool](1<<20, 64), + stop: make(chan struct{}), + } +} + +// Submit offers a hostname to the discovery engine. Duplicate registrable +// domains and already-tested candidate names are dropped cheaply. +func (d *Discoverer) Submit(host string) { + d.q.push(host) +} + +func (d *Discoverer) Start(ctx context.Context) { + go d.refill(ctx) + for i := 0; i < d.cfg.Workers; i++ { + d.wg.Add(1) + go d.worker(ctx) + } +} + +func (d *Discoverer) Close() { close(d.stop) } +func (d *Discoverer) Wait() { d.wg.Wait() } + +// QueueLen reports the sampled domain backlog. +func (d *Discoverer) QueueLen() int { return d.q.len() } + +func (d *Discoverer) refill(ctx context.Context) { + period := time.Second / time.Duration(d.cfg.QPS) + if period <= 0 { + period = time.Millisecond + } + t := time.NewTicker(period) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-d.stop: + return + case <-t.C: + select { + case d.tokens <- struct{}{}: + default: + } + } + } +} + +func (d *Discoverer) acquire(ctx context.Context) bool { + select { + case <-ctx.Done(): + return false + case <-d.stop: + return false + case <-d.tokens: + return true + } +} + +func (d *Discoverer) worker(ctx context.Context) { + defer d.wg.Done() + for { + select { + case <-ctx.Done(): + return + case <-d.stop: + return + case <-d.q.notify: + case <-d.names.notify: + } + for { + host, ok := d.q.pop() + if !ok { + break + } + d.scanDomain(ctx, host) + } + for { + name, ok := d.names.pop() + if !ok { + break + } + d.testName(ctx, name) + } + } +} + +func (d *Discoverer) scanDomain(ctx context.Context, host string) { + for _, name := range candidates(host, d.cfg.MaxPerDomain) { + if ctx.Err() != nil { + return + } + d.testName(ctx, name) + } +} + +// SubmitName tests a specific candidate bucket name (e.g. from a wordlist). +func (d *Discoverer) SubmitName(name string) { + if validBucketName(name) { + d.names.push(name) + } +} + +func (d *Discoverer) testName(ctx context.Context, name string) { + if _, dup := d.seen.Get(name); dup { + return + } + d.seen.Set(name, true, 12*time.Hour) + for _, t := range d.targets(name) { + if d.cfg.Metrics != nil { + d.cfg.Metrics.BucketCandidates.Add(1) + } + if !d.acquire(ctx) { + return + } + d.test(ctx, t) + } +} + +// Target is one provider listing endpoint for a candidate name. +type Target struct { + Name string + Provider model.Provider + Service model.Service + Hostname string // hostname for the indexer/probe candidate + URL string // anonymous list URL + Region string +} + +func (d *Discoverer) targets(name string) []Target { + var out []Target + if d.cfg.S3 { + out = append(out, Target{ + Name: name, Provider: model.ProviderAWS, Service: model.ServiceS3, + Hostname: name + ".s3.amazonaws.com", + URL: "https://" + name + ".s3.amazonaws.com/?list-type=2&max-keys=1", + }) + } + if d.cfg.GCS { + // Use the virtual-host form so each bucket has a unique hostname (the + // path form makes every bucket share storage.googleapis.com and collide + // on the cloud_assets unique key). + out = append(out, Target{ + Name: name, Provider: model.ProviderGCP, Service: model.ServiceObjectStorage, + Hostname: name + ".storage.googleapis.com", + URL: "https://storage.googleapis.com/" + name + "?max-keys=1", + }) + } + if d.cfg.DOSpaces { + for _, region := range d.cfg.DORegions { + host := name + "." + region + ".digitaloceanspaces.com" + out = append(out, Target{ + Name: name, Provider: model.ProviderDigitalOcean, Service: model.ServiceObjectStorage, + Hostname: host, Region: region, + URL: "https://" + host + "/?list-type=2&max-keys=1", + }) + } + } + return out +} + +func (d *Discoverer) test(ctx context.Context, t Target) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, t.URL, nil) + if err != nil { + return + } + resp, err := d.client.Do(req) + if err != nil { + return + } + _, _ = io.CopyN(io.Discard, resp.Body, 512) + finalHost := resp.Request.URL.Host + resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return + } + // Listable. Use the post-redirect host (S3 global redirects to a region). + c := model.Candidate{ + Hostname: t.Hostname, Original: t.Hostname, Resource: t.Name, Region: t.Region, + Provider: t.Provider, Service: t.Service, ResourceSpecific: true, Source: model.SourceDiscovery, + } + if finalHost != "" && strings.HasSuffix(finalHost, ".amazonaws.com") { + c.Hostname = finalHost + } + detail := "discovered open bucket via name enumeration (" + t.Hostname + ")" + if d.cfg.Metrics != nil { + d.cfg.Metrics.BucketOpen.Add(1) + } + if d.cfg.OnOpen == nil { + return + } + var entries []model.Entry + if d.cfg.Indexer != nil { + if e, err := d.cfg.Indexer.List(ctx, c); err == nil { + entries = e + } + } + d.cfg.OnOpen(ctx, c, detail, entries) +} + +// --- candidate generation --- + +var suffixes = []string{ + "", "-assets", "-static", "-backups", "-backup", "-images", "-img", "-media", + "-prod", "-dev", "-staging", "-test", "-files", "-uploads", "-public", "-cdn", + "-data", "-content", "-site", "-web", "-www", "-app", "-store", "-downloads", + "-photos", "-videos", "-docs", "-documents", "-archive", "-logs", "-artifacts", + "-build", "-deploy", "-release", "-db", "-database", "-sql", "-exports", "-reports", + "-assets-prod", "-assets-dev", "-static-prod", "-uploads-prod", +} + +// Wordlist is a bounded set of common global bucket names tested once each. +func Wordlist() []string { + return []string{ + "backups", "backup", "images", "image", "img", "media", "static", "assets", + "uploads", "files", "file", "data", "content", "public", "cdn", "www", "web", + "website", "app", "store", "downloads", "docs", "documents", "archive", + "logs", "artifacts", "build", "deploy", "release", "db", "database", "sql", + "dump", "exports", "reports", "photos", "videos", "dev", "test", "staging", + "prod", "production", "internal", "private", "secure", "userdata", "usercontent", + "thumbnails", "avatars", "profiles", "attachments", "tmp", "temp", "cache", + "config", "secrets", "keys", "certificates", "certs", "gifs", "png", "jpeg", + "download", "uploads-test", "uploads-dev", "assets-test", "assets-dev", + "static-assets", "web-assets", "app-assets", "site-assets", "media-assets", + } +} + +// baseLabels returns plausible bucket-name stems from a hostname: the leftmost +// label and the registrable-domain label. +func baseLabels(host string) []string { + host = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(host), ".")) + labels := strings.Split(host, ".") + if len(labels) < 2 { + return nil + } + cands := []string{labels[0], labels[len(labels)-2]} + seen := map[string]bool{} + out := cands[:0] + for _, c := range cands { + if validLabel(c) && !seen[c] && !skipLabel[c] { + seen[c] = true + out = append(out, c) + } + } + return out +} + +// Only genuinely non-descriptive labels are skipped; service-ish subdomains +// (assets, cdn, media, ...) are kept because they are common bucket names. +var skipLabel = map[string]bool{ + "www": true, "api": true, "mail": true, "email": true, "smtp": true, + "ns": true, "mx": true, "localhost": true, +} + +func candidates(host string, max int) []string { + var out []string + for _, b := range baseLabels(host) { + for _, s := range suffixes { + name := b + s + if validBucketName(name) { + out = append(out, name) + } + if max > 0 && len(out) >= max { + return out + } + } + } + return out +} + +func validLabel(s string) bool { + if len(s) < 3 || len(s) > 40 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') { + return false + } + } + return s[0] != '-' && s[len(s)-1] != '-' +} + +func validBucketName(s string) bool { + if len(s) < 3 || len(s) > 63 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '.') { + return false + } + } + return s[0] != '-' && s[0] != '.' && s[len(s)-1] != '-' && s[len(s)-1] != '.' +} + +// --- sampled domain queue --- + +type domainQueue struct { + mu sync.Mutex + items []string + cap int + notify chan struct{} +} + +func newDomainQueue(capacity int) *domainQueue { + return &domainQueue{cap: capacity, notify: make(chan struct{}, 1)} +} + +func (q *domainQueue) push(s string) { + q.mu.Lock() + if len(q.items) < q.cap { + q.items = append(q.items, s) + } else { + q.items[rand.IntN(len(q.items))] = s + } + q.mu.Unlock() + select { + case q.notify <- struct{}{}: + default: + } +} + +func (q *domainQueue) pop() (string, bool) { + q.mu.Lock() + n := len(q.items) + if n == 0 { + q.mu.Unlock() + return "", false + } + i := rand.IntN(n) + s := q.items[i] + q.items[i] = q.items[n-1] + q.items = q.items[:n-1] + q.mu.Unlock() + return s, true +} + +func (q *domainQueue) len() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.items) +} diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go new file mode 100644 index 0000000..bdb32fa --- /dev/null +++ b/internal/discover/discover_test.go @@ -0,0 +1,63 @@ +package discover + +import ( + "strings" + "testing" +) + +func TestCandidates(t *testing.T) { + got := candidates("assets.example.com", 200) + set := map[string]bool{} + for _, c := range got { + set[c] = true + } + for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} { + if !set[want] { + t.Fatalf("candidates missing %q (got %v)", want, got) + } + } + for _, c := range got { + if !validBucketName(c) { + t.Fatalf("invalid candidate %q", c) + } + } +} + +func TestBaseLabelsSkipsNoise(t *testing.T) { + // www is skipped, so only the registrable label remains. + got := baseLabels("www.example.com") + if len(got) != 1 || got[0] != "example" { + t.Fatalf("baseLabels = %v, want [example]", got) + } + if baseLabels("example") != nil { + t.Fatal("single label must yield no base") + } +} + +func TestTargets(t *testing.T) { + d := New(Config{S3: true, GCS: true, DOSpaces: true, DORegions: []string{"nyc3"}}) + ts := d.targets("example") + if len(ts) != 3 { + t.Fatalf("got %d targets, want 3", len(ts)) + } + var s3, gcs, do bool + for _, tg := range ts { + switch { + case strings.Contains(tg.URL, "example.s3.amazonaws.com/?list-type=2"): + s3 = true + case strings.Contains(tg.URL, "storage.googleapis.com/example"): + gcs = true + case strings.Contains(tg.URL, "example.nyc3.digitaloceanspaces.com/?list-type=2"): + do = true + } + } + if !s3 || !gcs || !do { + t.Fatalf("missing targets: s3=%v gcs=%v do=%v (%v)", s3, gcs, do, ts) + } +} + +func TestMaxPerDomain(t *testing.T) { + if got := candidates("example.com", 5); len(got) != 5 { + t.Fatalf("MaxPerDomain not honored: got %d", len(got)) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 47ba9fc..ddbbed4 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -41,6 +41,9 @@ type Metrics struct { ProbePublic atomic.Uint64 EntriesIndexed atomic.Uint64 + BucketCandidates atomic.Uint64 + BucketOpen atomic.Uint64 + gauges []gauge } @@ -86,6 +89,8 @@ func (m *Metrics) Handler() http.Handler { counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load()) counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load()) counter("entries_indexed_total", "Metadata entries indexed from readable resources", m.EntriesIndexed.Load()) + counter("bucket_candidates_total", "Candidate bucket names tested", m.BucketCandidates.Load()) + counter("bucket_open_total", "Open (listable) buckets discovered by name enumeration", m.BucketOpen.Load()) for _, g := range m.gauges { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn()) } diff --git a/internal/model/model.go b/internal/model/model.go index 5ca272a..113e53f 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -126,6 +126,7 @@ const ( SourceCNAME SourceDNSAlias SourceManual + SourceDiscovery ) func (s Source) String() string { @@ -138,6 +139,8 @@ func (s Source) String() string { return "dns_alias" case SourceManual: return "manual" + case SourceDiscovery: + return "discovery" default: return "unknown" } diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index ac91dfd..bb1de7c 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -42,6 +42,10 @@ type Config struct { Enricher *dns.Enricher Write WriteFunc Metrics *metrics.Metrics + + // Discover, when set, receives non-matching hostnames for bucket-name + // enumeration (see internal/discover). + Discover interface{ Submit(host string) } } // Pipeline is the certificate-to-database data flow. It is safe to call @@ -176,6 +180,9 @@ func (p *Pipeline) HandleHost(host string) { } p.dedup.Set(host, true, p.cfg.DedupTTL) p.dnsQ.push(host) + if p.cfg.Discover != nil { + p.cfg.Discover.Submit(host) + } } func (p *Pipeline) drop() {