make bucket-scan wordlist opt-in (default off)

Discovery is now driven purely by incoming CT domains (stem + suffix
expansion); the hardcoded generic-name wordlist is only used when
BUCKET_SCAN_WORDLIST=true.

.env.example

@@ -35,3 +35,4 @@ BUCKET_SCAN_QPS=50
 BUCKET_SCAN_WORKERS=16
 BUCKET_SCAN_QUEUE=65536
 BUCKET_SCAN_DO=false
+BUCKET_SCAN_WORDLIST=false

README.md

@@ -132,6 +132,7 @@ Environment variables only; no config files, no runtime pattern files.
 | `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers |
 | `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog |
 | `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions |
+| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names |
 | `DEDUP_TTL` | `1h` | hostname/finding dedup window |
 | `METRICS_ADDR` | `:9090` | metrics/health/UI listener |
 | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI |

cmd/ctcloud/main.go

@@ -167,12 +167,16 @@ func run() error {
 			},
 		})
 		discoverer.Start(ctx)
-		for _, name := range discover.Wordlist() {
+		wordlist := 0
-			discoverer.SubmitName(name)
+		if cfg.BucketScanWordlist {
+			for _, name := range discover.Wordlist() {
+				discoverer.SubmitName(name)
+			}
+			wordlist = len(discover.Wordlist())
 		}
 		slog.Info("bucket name discovery enabled", "event", "startup",
 			"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO,
-			"wordlist", len(discover.Wordlist()))
+			"wordlist", wordlist)
 	}
 
 	var prober *probe.Prober

internal/config/config.go

@@ -39,11 +39,12 @@ type Config struct {
 	IndexEnabled    bool
 	IndexMaxObjects int
 
-	BucketScanEnabled bool
+	BucketScanEnabled  bool
-	BucketScanQPS     int
+	BucketScanQPS      int
-	BucketScanWorkers int
+	BucketScanWorkers  int
-	BucketScanQueue   int
+	BucketScanQueue    int
-	BucketScanDO      bool
+	BucketScanDO       bool
+	BucketScanWordlist bool
 
 	MetricsAddr   string
 	UIUser        string
@@ -63,43 +64,44 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
 
 func Load() (*Config, error) {
 	c := &Config{
-		DatabaseURL:       os.Getenv("DATABASE_URL"),
+		DatabaseURL:        os.Getenv("DATABASE_URL"),
-		DBMaxConns:        envInt("DB_MAX_CONNS", 8),
+		DBMaxConns:         envInt("DB_MAX_CONNS", 8),
-		DNSServers:        envList("DNS_SERVERS", nil),
+		DNSServers:         envList("DNS_SERVERS", nil),
-		DNSWorkers:        envInt("DNS_WORKERS", 64),
+		DNSWorkers:         envInt("DNS_WORKERS", 64),
-		DNSTimeout:        envDuration("DNS_TIMEOUT", 5*time.Second),
+		DNSTimeout:         envDuration("DNS_TIMEOUT", 5*time.Second),
-		DNSCacheSize:      envInt("DNS_CACHE_SIZE", 1<<18),
+		DNSCacheSize:       envInt("DNS_CACHE_SIZE", 1<<18),
-		DedupSize:         envInt("DEDUP_SIZE", 1<<18),
+		DedupSize:          envInt("DEDUP_SIZE", 1<<18),
-		DNSMaxCNAMEdepth:  envInt("DNS_MAX_CNAME_DEPTH", 8),
+		DNSMaxCNAMEdepth:   envInt("DNS_MAX_CNAME_DEPTH", 8),
-		DNSPositiveTTL:    envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
+		DNSPositiveTTL:     envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
-		DNSNegativeTTL:    envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
+		DNSNegativeTTL:     envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
-		DirectQueueSize:   envInt("DIRECT_QUEUE_SIZE", 8192),
+		DirectQueueSize:    envInt("DIRECT_QUEUE_SIZE", 8192),
-		DNSQueueSize:      envInt("DNS_QUEUE_SIZE", 262144),
+		DNSQueueSize:       envInt("DNS_QUEUE_SIZE", 262144),
-		DBQueueSize:       envInt("DB_QUEUE_SIZE", 8192),
+		DBQueueSize:        envInt("DB_QUEUE_SIZE", 8192),
-		DBBatchSize:       envInt("DB_BATCH_SIZE", 500),
+		DBBatchSize:        envInt("DB_BATCH_SIZE", 500),
-		DBFlushInterval:   envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
+		DBFlushInterval:    envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
-		ActiveProbing:     envBool("ACTIVE_PROBING", false),
+		ActiveProbing:      envBool("ACTIVE_PROBING", false),
-		ProbeWorkers:      envInt("PROBE_WORKERS", 8),
+		ProbeWorkers:       envInt("PROBE_WORKERS", 8),
-		ProbeTimeout:      envDuration("PROBE_TIMEOUT", 5*time.Second),
+		ProbeTimeout:       envDuration("PROBE_TIMEOUT", 5*time.Second),
-		ProbeQueueSize:    envInt("PROBE_QUEUE_SIZE", 1<<16),
+		ProbeQueueSize:     envInt("PROBE_QUEUE_SIZE", 1<<16),
-		ProbeDeep:         envBool("PROBE_DEEP", true),
+		ProbeDeep:          envBool("PROBE_DEEP", true),
-		IndexEnabled:      envBool("INDEX_ENABLED", true),
+		IndexEnabled:       envBool("INDEX_ENABLED", true),
-		IndexMaxObjects:   envInt("INDEX_MAX_OBJECTS", 1000),
+		IndexMaxObjects:    envInt("INDEX_MAX_OBJECTS", 1000),
-		BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
+		BucketScanEnabled:  envBool("BUCKET_SCAN_ENABLED", true),
-		BucketScanQPS:     envInt("BUCKET_SCAN_QPS", 50),
+		BucketScanQPS:      envInt("BUCKET_SCAN_QPS", 50),
-		BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
+		BucketScanWorkers:  envInt("BUCKET_SCAN_WORKERS", 16),
-		BucketScanQueue:   envInt("BUCKET_SCAN_QUEUE", 65536),
+		BucketScanQueue:    envInt("BUCKET_SCAN_QUEUE", 65536),
-		BucketScanDO:      envBool("BUCKET_SCAN_DO", false),
+		BucketScanDO:       envBool("BUCKET_SCAN_DO", false),
-		MetricsAddr:       envString("METRICS_ADDR", ":9090"),
+		BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false),
-		UIUser:            envString("UI_USER", ""),
+		MetricsAddr:        envString("METRICS_ADDR", ":9090"),
-		UIPass:            envString("UI_PASS", ""),
+		UIUser:             envString("UI_USER", ""),
-		LogLevel:          envString("LOG_LEVEL", "info"),
+		UIPass:             envString("UI_PASS", ""),
-		LogFormat:         envString("LOG_FORMAT", "text"),
+		LogLevel:           envString("LOG_LEVEL", "info"),
-		StatsInterval:     envDuration("STATS_INTERVAL", 60*time.Second),
+		LogFormat:          envString("LOG_FORMAT", "text"),
-		LogListSource:     envString("LOG_LIST_SOURCE", DefaultLogListSource),
+		StatsInterval:      envDuration("STATS_INTERVAL", 60*time.Second),
-		StartAtEnd:        envBool("START_AT_END", false),
+		LogListSource:      envString("LOG_LIST_SOURCE", DefaultLogListSource),
-		ShutdownTimeout:   envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		StartAtEnd:         envBool("START_AT_END", false),
-		DedupeTTL:         envDuration("DEDUP_TTL", time.Hour),
+		ShutdownTimeout:    envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		DedupeTTL:          envDuration("DEDUP_TTL", time.Hour),
 	}
 	if err := c.validate(); err != nil {
 		return nil, err