bucket discovery engine: CT domain -> candidate names -> listing index

CMS SANs are custom domains, not bucket names, so matching SANs finds almost
no buckets. This adds internal/discover: from each non-matching hostname it
derives candidate names (domain label + subdomain label + ~40 suffixes) plus a
built-in wordlist, and anonymously tests listing on S3, GCS and (optionally)
DigitalOcean Spaces. Every 200 is indexed (keys/sizes/dates only).

Bounded hard: global BUCKET_SCAN_QPS rate limit, 12h per-name dedup, random
sampled domain queue, MaxPerDomain cap. Never downloads contents.

Fix: GCS candidates now use the virtual-host form
(<bucket>.storage.googleapis.com) so buckets no longer collide on the
cloud_assets unique key; previously they all shared storage.googleapis.com and
overwrote each other.

Live: 30 open buckets in ~5 min (e.g. estimator-dev, image-prod,
update-media, mos-images S3; several GCS buckets), 11k+ objects indexed.

internal/config/config.go

@@ -39,6 +39,12 @@ type Config struct {
 	IndexEnabled    bool
 	IndexMaxObjects int
 
+	BucketScanEnabled bool
+	BucketScanQPS     int
+	BucketScanWorkers int
+	BucketScanQueue   int
+	BucketScanDO      bool
+
 	MetricsAddr   string
 	UIUser        string
 	UIPass        string
@@ -57,38 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
 
 func Load() (*Config, error) {
 	c := &Config{
-		DatabaseURL:      os.Getenv("DATABASE_URL"),
-		DBMaxConns:       envInt("DB_MAX_CONNS", 8),
-		DNSServers:       envList("DNS_SERVERS", nil),
-		DNSWorkers:       envInt("DNS_WORKERS", 64),
-		DNSTimeout:       envDuration("DNS_TIMEOUT", 5*time.Second),
-		DNSCacheSize:     envInt("DNS_CACHE_SIZE", 1<<18),
-		DedupSize:        envInt("DEDUP_SIZE", 1<<18),
-		DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
-		DNSPositiveTTL:   envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
-		DNSNegativeTTL:   envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
-		DirectQueueSize:  envInt("DIRECT_QUEUE_SIZE", 8192),
-		DNSQueueSize:     envInt("DNS_QUEUE_SIZE", 262144),
-		DBQueueSize:      envInt("DB_QUEUE_SIZE", 8192),
-		DBBatchSize:      envInt("DB_BATCH_SIZE", 500),
-		DBFlushInterval:  envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
-		ActiveProbing:    envBool("ACTIVE_PROBING", false),
-		ProbeWorkers:     envInt("PROBE_WORKERS", 8),
-		ProbeTimeout:     envDuration("PROBE_TIMEOUT", 5*time.Second),
-		ProbeQueueSize:   envInt("PROBE_QUEUE_SIZE", 1<<16),
-		ProbeDeep:        envBool("PROBE_DEEP", true),
-		IndexEnabled:     envBool("INDEX_ENABLED", true),
-		IndexMaxObjects:  envInt("INDEX_MAX_OBJECTS", 1000),
-		MetricsAddr:      envString("METRICS_ADDR", ":9090"),
-		UIUser:           envString("UI_USER", ""),
-		UIPass:           envString("UI_PASS", ""),
-		LogLevel:         envString("LOG_LEVEL", "info"),
-		LogFormat:        envString("LOG_FORMAT", "text"),
-		StatsInterval:    envDuration("STATS_INTERVAL", 60*time.Second),
-		LogListSource:    envString("LOG_LIST_SOURCE", DefaultLogListSource),
-		StartAtEnd:       envBool("START_AT_END", false),
-		ShutdownTimeout:  envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
-		DedupeTTL:        envDuration("DEDUP_TTL", time.Hour),
+		DatabaseURL:       os.Getenv("DATABASE_URL"),
+		DBMaxConns:        envInt("DB_MAX_CONNS", 8),
+		DNSServers:        envList("DNS_SERVERS", nil),
+		DNSWorkers:        envInt("DNS_WORKERS", 64),
+		DNSTimeout:        envDuration("DNS_TIMEOUT", 5*time.Second),
+		DNSCacheSize:      envInt("DNS_CACHE_SIZE", 1<<18),
+		DedupSize:         envInt("DEDUP_SIZE", 1<<18),
+		DNSMaxCNAMEdepth:  envInt("DNS_MAX_CNAME_DEPTH", 8),
+		DNSPositiveTTL:    envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
+		DNSNegativeTTL:    envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
+		DirectQueueSize:   envInt("DIRECT_QUEUE_SIZE", 8192),
+		DNSQueueSize:      envInt("DNS_QUEUE_SIZE", 262144),
+		DBQueueSize:       envInt("DB_QUEUE_SIZE", 8192),
+		DBBatchSize:       envInt("DB_BATCH_SIZE", 500),
+		DBFlushInterval:   envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
+		ActiveProbing:     envBool("ACTIVE_PROBING", false),
+		ProbeWorkers:      envInt("PROBE_WORKERS", 8),
+		ProbeTimeout:      envDuration("PROBE_TIMEOUT", 5*time.Second),
+		ProbeQueueSize:    envInt("PROBE_QUEUE_SIZE", 1<<16),
+		ProbeDeep:         envBool("PROBE_DEEP", true),
+		IndexEnabled:      envBool("INDEX_ENABLED", true),
+		IndexMaxObjects:   envInt("INDEX_MAX_OBJECTS", 1000),
+		BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
+		BucketScanQPS:     envInt("BUCKET_SCAN_QPS", 50),
+		BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
+		BucketScanQueue:   envInt("BUCKET_SCAN_QUEUE", 65536),
+		BucketScanDO:      envBool("BUCKET_SCAN_DO", false),
+		MetricsAddr:       envString("METRICS_ADDR", ":9090"),
+		UIUser:            envString("UI_USER", ""),
+		UIPass:            envString("UI_PASS", ""),
+		LogLevel:          envString("LOG_LEVEL", "info"),
+		LogFormat:         envString("LOG_FORMAT", "text"),
+		StatsInterval:     envDuration("STATS_INTERVAL", 60*time.Second),
+		LogListSource:     envString("LOG_LIST_SOURCE", DefaultLogListSource),
+		StartAtEnd:        envBool("START_AT_END", false),
+		ShutdownTimeout:   envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		DedupeTTL:         envDuration("DEDUP_TTL", time.Hour),
 	}
 	if err := c.validate(); err != nil {
 		return nil, err