karl/bucketeer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make bucket-scan wordlist opt-in (default off)

Browse Source 
Discovery is now driven purely by incoming CT domains (stem + suffix
expansion); the hardcoded generic-name wordlist is only used when
BUCKET_SCAN_WORDLIST=true.
This commit is contained in:
StasonJatham committed 2026-09-22 16:04:47 +02:00
1 parent a899362e18
commit ebb020c6b7
4 files changed
+53 -45
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/config/config.go
+44 -42
View File
@@ -39,11 +39,12 @@ type Config struct {
IndexEnabled bool
IndexMaxObjects int
BucketScanEnabled bool
BucketScanQPS int
BucketScanWorkers int
BucketScanQueue int
BucketScanDO bool
BucketScanEnabled bool
BucketScanQPS int
BucketScanWorkers int
BucketScanQueue int
BucketScanDO bool
BucketScanWordlist bool
MetricsAddr string
UIUser string
@@ -63,43 +64,44 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
func Load() (*Config, error) {
c := &Config{
DatabaseURL: os.Getenv("DATABASE_URL"),
DBMaxConns: envInt("DB_MAX_CONNS", 8),
DNSServers: envList("DNS_SERVERS", nil),
DNSWorkers: envInt("DNS_WORKERS", 64),
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
DedupSize: envInt("DEDUP_SIZE", 1<<18),
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
ActiveProbing: envBool("ACTIVE_PROBING", false),
ProbeWorkers: envInt("PROBE_WORKERS", 8),
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
ProbeDeep: envBool("PROBE_DEEP", true),
IndexEnabled: envBool("INDEX_ENABLED", true),
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
BucketScanDO: envBool("BUCKET_SCAN_DO", false),
MetricsAddr: envString("METRICS_ADDR", ":9090"),
UIUser: envString("UI_USER", ""),
UIPass: envString("UI_PASS", ""),
LogLevel: envString("LOG_LEVEL", "info"),
LogFormat: envString("LOG_FORMAT", "text"),
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
StartAtEnd: envBool("START_AT_END", false),
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
DatabaseURL: os.Getenv("DATABASE_URL"),
DBMaxConns: envInt("DB_MAX_CONNS", 8),
DNSServers: envList("DNS_SERVERS", nil),
DNSWorkers: envInt("DNS_WORKERS", 64),
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
DedupSize: envInt("DEDUP_SIZE", 1<<18),
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
ActiveProbing: envBool("ACTIVE_PROBING", false),
ProbeWorkers: envInt("PROBE_WORKERS", 8),
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
ProbeDeep: envBool("PROBE_DEEP", true),
IndexEnabled: envBool("INDEX_ENABLED", true),
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
BucketScanDO: envBool("BUCKET_SCAN_DO", false),
BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false),
MetricsAddr: envString("METRICS_ADDR", ":9090"),
UIUser: envString("UI_USER", ""),
UIPass: envString("UI_PASS", ""),
LogLevel: envString("LOG_LEVEL", "info"),
LogFormat: envString("LOG_FORMAT", "text"),
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
StartAtEnd: envBool("START_AT_END", false),
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
}
if err := c.validate(); err != nil {
return nil, err