bucket discovery: no name guessing

Remove all fabricated candidate names (suffixes and the wordlist). Discovery now
only tests names that actually occur in the incoming hostname: the subdomain
label and the registrable-domain label (assets.example.com -> assets, example).
Nothing is permuted or drawn from a wordlist.

.env.example

@@ -35,4 +35,3 @@ BUCKET_SCAN_QPS=50
 BUCKET_SCAN_WORKERS=16
 BUCKET_SCAN_QUEUE=65536
 BUCKET_SCAN_DO=false
-BUCKET_SCAN_WORDLIST=false

README.md

@@ -132,7 +132,6 @@ Environment variables only; no config files, no runtime pattern files.
 | `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers |
 | `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog |
 | `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions |
-| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names |
 | `DEDUP_TTL` | `1h` | hostname/finding dedup window |
 | `METRICS_ADDR` | `:9090` | metrics/health/UI listener |
 | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI |
@@ -317,9 +316,9 @@ Certificate SANs are custom domains, not bucket names, so matching SANs alone
 finds almost no buckets. Instead, every non-matching hostname feeds a discovery
 engine that:
 
-1. derives candidate names from the domain (`example` → `example`,
-   `example-assets`, `example-backups`, …, plus the subdomain label), also
-   testing a built-in wordlist of common names;
+1. takes the names that actually occur in the hostname — the subdomain label
+   and the registrable-domain label (`assets.example.com` → `assets`,
+   `example`). No names are fabricated, permuted, or drawn from a wordlist;
 2. tests anonymous listing per provider —
    `https://<name>.s3.amazonaws.com/?list-type=2` (AWS),
    `https://<name>.storage.googleapis.com` (GCS),

cmd/ctcloud/main.go

@@ -167,16 +167,8 @@ func run() error {
 			},
 		})
 		discoverer.Start(ctx)
-		wordlist := 0
-		if cfg.BucketScanWordlist {
-			for _, name := range discover.Wordlist() {
-				discoverer.SubmitName(name)
-			}
-			wordlist = len(discover.Wordlist())
-		}
 		slog.Info("bucket name discovery enabled", "event", "startup",
-			"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO,
-			"wordlist", wordlist)
+			"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO)
 	}
 
 	var prober *probe.Prober

internal/config/config.go

@@ -39,12 +39,11 @@ type Config struct {
 	IndexEnabled    bool
 	IndexMaxObjects int
 
-	BucketScanEnabled  bool
-	BucketScanQPS      int
-	BucketScanWorkers  int
-	BucketScanQueue    int
-	BucketScanDO       bool
-	BucketScanWordlist bool
+	BucketScanEnabled bool
+	BucketScanQPS     int
+	BucketScanWorkers int
+	BucketScanQueue   int
+	BucketScanDO      bool
 
 	MetricsAddr   string
 	UIUser        string
@@ -64,44 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
 
 func Load() (*Config, error) {
 	c := &Config{
-		DatabaseURL:        os.Getenv("DATABASE_URL"),
-		DBMaxConns:         envInt("DB_MAX_CONNS", 8),
-		DNSServers:         envList("DNS_SERVERS", nil),
-		DNSWorkers:         envInt("DNS_WORKERS", 64),
-		DNSTimeout:         envDuration("DNS_TIMEOUT", 5*time.Second),
-		DNSCacheSize:       envInt("DNS_CACHE_SIZE", 1<<18),
-		DedupSize:          envInt("DEDUP_SIZE", 1<<18),
-		DNSMaxCNAMEdepth:   envInt("DNS_MAX_CNAME_DEPTH", 8),
-		DNSPositiveTTL:     envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
-		DNSNegativeTTL:     envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
-		DirectQueueSize:    envInt("DIRECT_QUEUE_SIZE", 8192),
-		DNSQueueSize:       envInt("DNS_QUEUE_SIZE", 262144),
-		DBQueueSize:        envInt("DB_QUEUE_SIZE", 8192),
-		DBBatchSize:        envInt("DB_BATCH_SIZE", 500),
-		DBFlushInterval:    envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
-		ActiveProbing:      envBool("ACTIVE_PROBING", false),
-		ProbeWorkers:       envInt("PROBE_WORKERS", 8),
-		ProbeTimeout:       envDuration("PROBE_TIMEOUT", 5*time.Second),
-		ProbeQueueSize:     envInt("PROBE_QUEUE_SIZE", 1<<16),
-		ProbeDeep:          envBool("PROBE_DEEP", true),
-		IndexEnabled:       envBool("INDEX_ENABLED", true),
-		IndexMaxObjects:    envInt("INDEX_MAX_OBJECTS", 1000),
-		BucketScanEnabled:  envBool("BUCKET_SCAN_ENABLED", true),
-		BucketScanQPS:      envInt("BUCKET_SCAN_QPS", 50),
-		BucketScanWorkers:  envInt("BUCKET_SCAN_WORKERS", 16),
-		BucketScanQueue:    envInt("BUCKET_SCAN_QUEUE", 65536),
-		BucketScanDO:       envBool("BUCKET_SCAN_DO", false),
-		BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false),
-		MetricsAddr:        envString("METRICS_ADDR", ":9090"),
-		UIUser:             envString("UI_USER", ""),
-		UIPass:             envString("UI_PASS", ""),
-		LogLevel:           envString("LOG_LEVEL", "info"),
-		LogFormat:          envString("LOG_FORMAT", "text"),
-		StatsInterval:      envDuration("STATS_INTERVAL", 60*time.Second),
-		LogListSource:      envString("LOG_LIST_SOURCE", DefaultLogListSource),
-		StartAtEnd:         envBool("START_AT_END", false),
-		ShutdownTimeout:    envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
-		DedupeTTL:          envDuration("DEDUP_TTL", time.Hour),
+		DatabaseURL:       os.Getenv("DATABASE_URL"),
+		DBMaxConns:        envInt("DB_MAX_CONNS", 8),
+		DNSServers:        envList("DNS_SERVERS", nil),
+		DNSWorkers:        envInt("DNS_WORKERS", 64),
+		DNSTimeout:        envDuration("DNS_TIMEOUT", 5*time.Second),
+		DNSCacheSize:      envInt("DNS_CACHE_SIZE", 1<<18),
+		DedupSize:         envInt("DEDUP_SIZE", 1<<18),
+		DNSMaxCNAMEdepth:  envInt("DNS_MAX_CNAME_DEPTH", 8),
+		DNSPositiveTTL:    envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
+		DNSNegativeTTL:    envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
+		DirectQueueSize:   envInt("DIRECT_QUEUE_SIZE", 8192),
+		DNSQueueSize:      envInt("DNS_QUEUE_SIZE", 262144),
+		DBQueueSize:       envInt("DB_QUEUE_SIZE", 8192),
+		DBBatchSize:       envInt("DB_BATCH_SIZE", 500),
+		DBFlushInterval:   envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
+		ActiveProbing:     envBool("ACTIVE_PROBING", false),
+		ProbeWorkers:      envInt("PROBE_WORKERS", 8),
+		ProbeTimeout:      envDuration("PROBE_TIMEOUT", 5*time.Second),
+		ProbeQueueSize:    envInt("PROBE_QUEUE_SIZE", 1<<16),
+		ProbeDeep:         envBool("PROBE_DEEP", true),
+		IndexEnabled:      envBool("INDEX_ENABLED", true),
+		IndexMaxObjects:   envInt("INDEX_MAX_OBJECTS", 1000),
+		BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
+		BucketScanQPS:     envInt("BUCKET_SCAN_QPS", 50),
+		BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
+		BucketScanQueue:   envInt("BUCKET_SCAN_QUEUE", 65536),
+		BucketScanDO:      envBool("BUCKET_SCAN_DO", false),
+		MetricsAddr:       envString("METRICS_ADDR", ":9090"),
+		UIUser:            envString("UI_USER", ""),
+		UIPass:            envString("UI_PASS", ""),
+		LogLevel:          envString("LOG_LEVEL", "info"),
+		LogFormat:         envString("LOG_FORMAT", "text"),
+		StatsInterval:     envDuration("STATS_INTERVAL", 60*time.Second),
+		LogListSource:     envString("LOG_LIST_SOURCE", DefaultLogListSource),
+		StartAtEnd:        envBool("START_AT_END", false),
+		ShutdownTimeout:   envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		DedupeTTL:         envDuration("DEDUP_TTL", time.Hour),
 	}
 	if err := c.validate(); err != nil {
 		return nil, err

internal/discover/discover.go

@@ -45,7 +45,6 @@ type Discoverer struct {
 	cfg    Config
 	client *http.Client
 	q      *domainQueue
-	names  *domainQueue
 	tokens chan struct{}
 	seen   *dns.TTLCache[bool] // candidate names already tested
 	stop   chan struct{}
@@ -69,7 +68,6 @@ func New(cfg Config) *Discoverer {
 		cfg:    cfg,
 		client: &http.Client{Timeout: 6 * time.Second},
 		q:      newDomainQueue(cfg.Queue),
-		names:  newDomainQueue(cfg.Queue),
 		tokens: make(chan struct{}, max(cfg.QPS, 1)),
 		seen:   dns.NewTTLCache[bool](1<<20, 64),
 		stop:   make(chan struct{}),
@@ -138,7 +136,6 @@ func (d *Discoverer) worker(ctx context.Context) {
 		case <-d.stop:
 			return
 		case <-d.q.notify:
-		case <-d.names.notify:
 		}
 		for {
 			host, ok := d.q.pop()
@@ -147,13 +144,6 @@ func (d *Discoverer) worker(ctx context.Context) {
 			}
 			d.scanDomain(ctx, host)
 		}
-		for {
-			name, ok := d.names.pop()
-			if !ok {
-				break
-			}
-			d.testName(ctx, name)
-		}
 	}
 }
 
@@ -166,13 +156,6 @@ func (d *Discoverer) scanDomain(ctx context.Context, host string) {
 	}
 }
 
-// SubmitName tests a specific candidate bucket name (e.g. from a wordlist).
-func (d *Discoverer) SubmitName(name string) {
-	if validBucketName(name) {
-		d.names.push(name)
-	}
-}
-
 func (d *Discoverer) testName(ctx context.Context, name string) {
 	if _, dup := d.seen.Get(name); dup {
 		return
@@ -272,30 +255,9 @@ func (d *Discoverer) test(ctx context.Context, t Target) {
 
 // --- candidate generation ---
 
-var suffixes = []string{
-	"", "-assets", "-static", "-backups", "-backup", "-images", "-img", "-media",
-	"-prod", "-dev", "-staging", "-test", "-files", "-uploads", "-public", "-cdn",
-	"-data", "-content", "-site", "-web", "-www", "-app", "-store", "-downloads",
-	"-photos", "-videos", "-docs", "-documents", "-archive", "-logs", "-artifacts",
-	"-build", "-deploy", "-release", "-db", "-database", "-sql", "-exports", "-reports",
-	"-assets-prod", "-assets-dev", "-static-prod", "-uploads-prod",
-}
-
-// Wordlist is a bounded set of common global bucket names tested once each.
-func Wordlist() []string {
-	return []string{
-		"backups", "backup", "images", "image", "img", "media", "static", "assets",
-		"uploads", "files", "file", "data", "content", "public", "cdn", "www", "web",
-		"website", "app", "store", "downloads", "docs", "documents", "archive",
-		"logs", "artifacts", "build", "deploy", "release", "db", "database", "sql",
-		"dump", "exports", "reports", "photos", "videos", "dev", "test", "staging",
-		"prod", "production", "internal", "private", "secure", "userdata", "usercontent",
-		"thumbnails", "avatars", "profiles", "attachments", "tmp", "temp", "cache",
-		"config", "secrets", "keys", "certificates", "certs", "gifs", "png", "jpeg",
-		"download", "uploads-test", "uploads-dev", "assets-test", "assets-dev",
-		"static-assets", "web-assets", "app-assets", "site-assets", "media-assets",
-	}
-}
+// No name guessing: candidates are exactly the labels that occur in the incoming
+// hostname (the subdomain label and the registrable-domain label). Nothing is
+// fabricated, permuted, or drawn from a wordlist.
 
 // baseLabels returns plausible bucket-name stems from a hostname: the leftmost
 // label and the registrable-domain label.
@@ -325,17 +287,9 @@ var skipLabel = map[string]bool{
 }
 
 func candidates(host string, max int) []string {
-	var out []string
-	for _, b := range baseLabels(host) {
-		for _, s := range suffixes {
-			name := b + s
-			if validBucketName(name) {
-				out = append(out, name)
-			}
-			if max > 0 && len(out) >= max {
-				return out
-			}
-		}
+	out := baseLabels(host)
+	if max > 0 && len(out) > max {
+		out = out[:max]
 	}
 	return out
 }

internal/discover/discover_test.go

@@ -5,16 +5,10 @@ import (
 	"testing"
 )
 
-func TestCandidates(t *testing.T) {
+func TestCandidatesAreExactLabelsOnly(t *testing.T) {
 	got := candidates("assets.example.com", 200)
-	set := map[string]bool{}
-	for _, c := range got {
-		set[c] = true
-	}
-	for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} {
-		if !set[want] {
-			t.Fatalf("candidates missing %q (got %v)", want, got)
-		}
+	if len(got) != 2 || got[0] != "assets" || got[1] != "example" {
+		t.Fatalf("candidates = %v, want [assets example] (no fabricated names)", got)
 	}
 	for _, c := range got {
 		if !validBucketName(c) {
@@ -24,7 +18,6 @@ func TestCandidates(t *testing.T) {
 }
 
 func TestBaseLabelsSkipsNoise(t *testing.T) {
-	// www is skipped, so only the registrable label remains.
 	got := baseLabels("www.example.com")
 	if len(got) != 1 || got[0] != "example" {
 		t.Fatalf("baseLabels = %v, want [example]", got)
@@ -57,7 +50,7 @@ func TestTargets(t *testing.T) {
 }
 
 func TestMaxPerDomain(t *testing.T) {
-	if got := candidates("example.com", 5); len(got) != 5 {
+	if got := candidates("example.com", 5); len(got) > 5 {
 		t.Fatalf("MaxPerDomain not honored: got %d", len(got))
 	}
 }