diff --git a/.env.example b/.env.example index db15e46..0822c33 100644 --- a/.env.example +++ b/.env.example @@ -35,4 +35,3 @@ BUCKET_SCAN_QPS=50 BUCKET_SCAN_WORKERS=16 BUCKET_SCAN_QUEUE=65536 BUCKET_SCAN_DO=false -BUCKET_SCAN_WORDLIST=false diff --git a/README.md b/README.md index 712f488..ca16036 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,6 @@ Environment variables only; no config files, no runtime pattern files. | `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers | | `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog | | `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions | -| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names | | `DEDUP_TTL` | `1h` | hostname/finding dedup window | | `METRICS_ADDR` | `:9090` | metrics/health/UI listener | | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI | @@ -317,9 +316,9 @@ Certificate SANs are custom domains, not bucket names, so matching SANs alone finds almost no buckets. Instead, every non-matching hostname feeds a discovery engine that: -1. derives candidate names from the domain (`example` → `example`, - `example-assets`, `example-backups`, …, plus the subdomain label), also - testing a built-in wordlist of common names; +1. takes the names that actually occur in the hostname — the subdomain label + and the registrable-domain label (`assets.example.com` → `assets`, + `example`). No names are fabricated, permuted, or drawn from a wordlist; 2. tests anonymous listing per provider — `https://.s3.amazonaws.com/?list-type=2` (AWS), `https://.storage.googleapis.com` (GCS), diff --git a/cmd/ctcloud/main.go b/cmd/ctcloud/main.go index 05e099f..cd73453 100644 --- a/cmd/ctcloud/main.go +++ b/cmd/ctcloud/main.go @@ -167,16 +167,8 @@ func run() error { }, }) discoverer.Start(ctx) - wordlist := 0 - if cfg.BucketScanWordlist { - for _, name := range discover.Wordlist() { - discoverer.SubmitName(name) - } - wordlist = len(discover.Wordlist()) - } slog.Info("bucket name discovery enabled", "event", "startup", - "qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO, - "wordlist", wordlist) + "qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO) } var prober *probe.Prober diff --git a/internal/config/config.go b/internal/config/config.go index c9a5c7f..993bc20 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,12 +39,11 @@ type Config struct { IndexEnabled bool IndexMaxObjects int - BucketScanEnabled bool - BucketScanQPS int - BucketScanWorkers int - BucketScanQueue int - BucketScanDO bool - BucketScanWordlist bool + BucketScanEnabled bool + BucketScanQPS int + BucketScanWorkers int + BucketScanQueue int + BucketScanDO bool MetricsAddr string UIUser string @@ -64,44 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js func Load() (*Config, error) { c := &Config{ - DatabaseURL: os.Getenv("DATABASE_URL"), - DBMaxConns: envInt("DB_MAX_CONNS", 8), - DNSServers: envList("DNS_SERVERS", nil), - DNSWorkers: envInt("DNS_WORKERS", 64), - DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), - DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), - DedupSize: envInt("DEDUP_SIZE", 1<<18), - DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), - DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), - DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), - DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), - DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), - DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), - DBBatchSize: envInt("DB_BATCH_SIZE", 500), - DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), - ActiveProbing: envBool("ACTIVE_PROBING", false), - ProbeWorkers: envInt("PROBE_WORKERS", 8), - ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), - ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), - ProbeDeep: envBool("PROBE_DEEP", true), - IndexEnabled: envBool("INDEX_ENABLED", true), - IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), - BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true), - BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50), - BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16), - BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536), - BucketScanDO: envBool("BUCKET_SCAN_DO", false), - BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false), - MetricsAddr: envString("METRICS_ADDR", ":9090"), - UIUser: envString("UI_USER", ""), - UIPass: envString("UI_PASS", ""), - LogLevel: envString("LOG_LEVEL", "info"), - LogFormat: envString("LOG_FORMAT", "text"), - StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), - LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), - StartAtEnd: envBool("START_AT_END", false), - ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), - DedupeTTL: envDuration("DEDUP_TTL", time.Hour), + DatabaseURL: os.Getenv("DATABASE_URL"), + DBMaxConns: envInt("DB_MAX_CONNS", 8), + DNSServers: envList("DNS_SERVERS", nil), + DNSWorkers: envInt("DNS_WORKERS", 64), + DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), + DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), + DedupSize: envInt("DEDUP_SIZE", 1<<18), + DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), + DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), + DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), + DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), + DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), + DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), + DBBatchSize: envInt("DB_BATCH_SIZE", 500), + DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), + ActiveProbing: envBool("ACTIVE_PROBING", false), + ProbeWorkers: envInt("PROBE_WORKERS", 8), + ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), + ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), + ProbeDeep: envBool("PROBE_DEEP", true), + IndexEnabled: envBool("INDEX_ENABLED", true), + IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), + BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true), + BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50), + BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16), + BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536), + BucketScanDO: envBool("BUCKET_SCAN_DO", false), + MetricsAddr: envString("METRICS_ADDR", ":9090"), + UIUser: envString("UI_USER", ""), + UIPass: envString("UI_PASS", ""), + LogLevel: envString("LOG_LEVEL", "info"), + LogFormat: envString("LOG_FORMAT", "text"), + StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), + LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), + StartAtEnd: envBool("START_AT_END", false), + ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), + DedupeTTL: envDuration("DEDUP_TTL", time.Hour), } if err := c.validate(); err != nil { return nil, err diff --git a/internal/discover/discover.go b/internal/discover/discover.go index da6caf7..433a9a2 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -45,7 +45,6 @@ type Discoverer struct { cfg Config client *http.Client q *domainQueue - names *domainQueue tokens chan struct{} seen *dns.TTLCache[bool] // candidate names already tested stop chan struct{} @@ -69,7 +68,6 @@ func New(cfg Config) *Discoverer { cfg: cfg, client: &http.Client{Timeout: 6 * time.Second}, q: newDomainQueue(cfg.Queue), - names: newDomainQueue(cfg.Queue), tokens: make(chan struct{}, max(cfg.QPS, 1)), seen: dns.NewTTLCache[bool](1<<20, 64), stop: make(chan struct{}), @@ -138,7 +136,6 @@ func (d *Discoverer) worker(ctx context.Context) { case <-d.stop: return case <-d.q.notify: - case <-d.names.notify: } for { host, ok := d.q.pop() @@ -147,13 +144,6 @@ func (d *Discoverer) worker(ctx context.Context) { } d.scanDomain(ctx, host) } - for { - name, ok := d.names.pop() - if !ok { - break - } - d.testName(ctx, name) - } } } @@ -166,13 +156,6 @@ func (d *Discoverer) scanDomain(ctx context.Context, host string) { } } -// SubmitName tests a specific candidate bucket name (e.g. from a wordlist). -func (d *Discoverer) SubmitName(name string) { - if validBucketName(name) { - d.names.push(name) - } -} - func (d *Discoverer) testName(ctx context.Context, name string) { if _, dup := d.seen.Get(name); dup { return @@ -272,30 +255,9 @@ func (d *Discoverer) test(ctx context.Context, t Target) { // --- candidate generation --- -var suffixes = []string{ - "", "-assets", "-static", "-backups", "-backup", "-images", "-img", "-media", - "-prod", "-dev", "-staging", "-test", "-files", "-uploads", "-public", "-cdn", - "-data", "-content", "-site", "-web", "-www", "-app", "-store", "-downloads", - "-photos", "-videos", "-docs", "-documents", "-archive", "-logs", "-artifacts", - "-build", "-deploy", "-release", "-db", "-database", "-sql", "-exports", "-reports", - "-assets-prod", "-assets-dev", "-static-prod", "-uploads-prod", -} - -// Wordlist is a bounded set of common global bucket names tested once each. -func Wordlist() []string { - return []string{ - "backups", "backup", "images", "image", "img", "media", "static", "assets", - "uploads", "files", "file", "data", "content", "public", "cdn", "www", "web", - "website", "app", "store", "downloads", "docs", "documents", "archive", - "logs", "artifacts", "build", "deploy", "release", "db", "database", "sql", - "dump", "exports", "reports", "photos", "videos", "dev", "test", "staging", - "prod", "production", "internal", "private", "secure", "userdata", "usercontent", - "thumbnails", "avatars", "profiles", "attachments", "tmp", "temp", "cache", - "config", "secrets", "keys", "certificates", "certs", "gifs", "png", "jpeg", - "download", "uploads-test", "uploads-dev", "assets-test", "assets-dev", - "static-assets", "web-assets", "app-assets", "site-assets", "media-assets", - } -} +// No name guessing: candidates are exactly the labels that occur in the incoming +// hostname (the subdomain label and the registrable-domain label). Nothing is +// fabricated, permuted, or drawn from a wordlist. // baseLabels returns plausible bucket-name stems from a hostname: the leftmost // label and the registrable-domain label. @@ -325,17 +287,9 @@ var skipLabel = map[string]bool{ } func candidates(host string, max int) []string { - var out []string - for _, b := range baseLabels(host) { - for _, s := range suffixes { - name := b + s - if validBucketName(name) { - out = append(out, name) - } - if max > 0 && len(out) >= max { - return out - } - } + out := baseLabels(host) + if max > 0 && len(out) > max { + out = out[:max] } return out } diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go index bdb32fa..989f937 100644 --- a/internal/discover/discover_test.go +++ b/internal/discover/discover_test.go @@ -5,16 +5,10 @@ import ( "testing" ) -func TestCandidates(t *testing.T) { +func TestCandidatesAreExactLabelsOnly(t *testing.T) { got := candidates("assets.example.com", 200) - set := map[string]bool{} - for _, c := range got { - set[c] = true - } - for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} { - if !set[want] { - t.Fatalf("candidates missing %q (got %v)", want, got) - } + if len(got) != 2 || got[0] != "assets" || got[1] != "example" { + t.Fatalf("candidates = %v, want [assets example] (no fabricated names)", got) } for _, c := range got { if !validBucketName(c) { @@ -24,7 +18,6 @@ func TestCandidates(t *testing.T) { } func TestBaseLabelsSkipsNoise(t *testing.T) { - // www is skipped, so only the registrable label remains. got := baseLabels("www.example.com") if len(got) != 1 || got[0] != "example" { t.Fatalf("baseLabels = %v, want [example]", got) @@ -57,7 +50,7 @@ func TestTargets(t *testing.T) { } func TestMaxPerDomain(t *testing.T) { - if got := candidates("example.com", 5); len(got) != 5 { + if got := candidates("example.com", 5); len(got) > 5 { t.Fatalf("MaxPerDomain not honored: got %d", len(got)) } }