From ebb020c6b71d7508ec1244e9727a0b5faf55a5c4 Mon Sep 17 00:00:00 2001 From: StasonJatham Date: Tue, 22 Sep 2026 16:04:47 +0200 Subject: [PATCH] make bucket-scan wordlist opt-in (default off) Discovery is now driven purely by incoming CT domains (stem + suffix expansion); the hardcoded generic-name wordlist is only used when BUCKET_SCAN_WORDLIST=true. --- .env.example | 1 + README.md | 1 + cmd/ctcloud/main.go | 10 +++-- internal/config/config.go | 86 ++++++++++++++++++++------------------- 4 files changed, 53 insertions(+), 45 deletions(-) diff --git a/.env.example b/.env.example index 0822c33..db15e46 100644 --- a/.env.example +++ b/.env.example @@ -35,3 +35,4 @@ BUCKET_SCAN_QPS=50 BUCKET_SCAN_WORKERS=16 BUCKET_SCAN_QUEUE=65536 BUCKET_SCAN_DO=false +BUCKET_SCAN_WORDLIST=false diff --git a/README.md b/README.md index c4b36c6..712f488 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ Environment variables only; no config files, no runtime pattern files. | `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers | | `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog | | `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions | +| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names | | `DEDUP_TTL` | `1h` | hostname/finding dedup window | | `METRICS_ADDR` | `:9090` | metrics/health/UI listener | | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI | diff --git a/cmd/ctcloud/main.go b/cmd/ctcloud/main.go index 01112a2..05e099f 100644 --- a/cmd/ctcloud/main.go +++ b/cmd/ctcloud/main.go @@ -167,12 +167,16 @@ func run() error { }, }) discoverer.Start(ctx) - for _, name := range discover.Wordlist() { - discoverer.SubmitName(name) + wordlist := 0 + if cfg.BucketScanWordlist { + for _, name := range discover.Wordlist() { + discoverer.SubmitName(name) + } + wordlist = len(discover.Wordlist()) } slog.Info("bucket name discovery enabled", "event", "startup", "qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO, - "wordlist", len(discover.Wordlist())) + "wordlist", wordlist) } var prober *probe.Prober diff --git a/internal/config/config.go b/internal/config/config.go index 993bc20..c9a5c7f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,11 +39,12 @@ type Config struct { IndexEnabled bool IndexMaxObjects int - BucketScanEnabled bool - BucketScanQPS int - BucketScanWorkers int - BucketScanQueue int - BucketScanDO bool + BucketScanEnabled bool + BucketScanQPS int + BucketScanWorkers int + BucketScanQueue int + BucketScanDO bool + BucketScanWordlist bool MetricsAddr string UIUser string @@ -63,43 +64,44 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js func Load() (*Config, error) { c := &Config{ - DatabaseURL: os.Getenv("DATABASE_URL"), - DBMaxConns: envInt("DB_MAX_CONNS", 8), - DNSServers: envList("DNS_SERVERS", nil), - DNSWorkers: envInt("DNS_WORKERS", 64), - DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), - DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), - DedupSize: envInt("DEDUP_SIZE", 1<<18), - DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), - DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), - DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), - DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), - DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), - DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), - DBBatchSize: envInt("DB_BATCH_SIZE", 500), - DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), - ActiveProbing: envBool("ACTIVE_PROBING", false), - ProbeWorkers: envInt("PROBE_WORKERS", 8), - ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), - ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), - ProbeDeep: envBool("PROBE_DEEP", true), - IndexEnabled: envBool("INDEX_ENABLED", true), - IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), - BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true), - BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50), - BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16), - BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536), - BucketScanDO: envBool("BUCKET_SCAN_DO", false), - MetricsAddr: envString("METRICS_ADDR", ":9090"), - UIUser: envString("UI_USER", ""), - UIPass: envString("UI_PASS", ""), - LogLevel: envString("LOG_LEVEL", "info"), - LogFormat: envString("LOG_FORMAT", "text"), - StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), - LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), - StartAtEnd: envBool("START_AT_END", false), - ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), - DedupeTTL: envDuration("DEDUP_TTL", time.Hour), + DatabaseURL: os.Getenv("DATABASE_URL"), + DBMaxConns: envInt("DB_MAX_CONNS", 8), + DNSServers: envList("DNS_SERVERS", nil), + DNSWorkers: envInt("DNS_WORKERS", 64), + DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), + DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), + DedupSize: envInt("DEDUP_SIZE", 1<<18), + DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), + DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), + DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), + DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), + DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), + DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), + DBBatchSize: envInt("DB_BATCH_SIZE", 500), + DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), + ActiveProbing: envBool("ACTIVE_PROBING", false), + ProbeWorkers: envInt("PROBE_WORKERS", 8), + ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), + ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), + ProbeDeep: envBool("PROBE_DEEP", true), + IndexEnabled: envBool("INDEX_ENABLED", true), + IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), + BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true), + BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50), + BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16), + BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536), + BucketScanDO: envBool("BUCKET_SCAN_DO", false), + BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false), + MetricsAddr: envString("METRICS_ADDR", ":9090"), + UIUser: envString("UI_USER", ""), + UIPass: envString("UI_PASS", ""), + LogLevel: envString("LOG_LEVEL", "info"), + LogFormat: envString("LOG_FORMAT", "text"), + StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), + LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), + StartAtEnd: envBool("START_AT_END", false), + ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), + DedupeTTL: envDuration("DEDUP_TTL", time.Hour), } if err := c.validate(); err != nil { return nil, err