make bucket-scan wordlist opt-in (default off)
Discovery is now driven purely by incoming CT domains (stem + suffix expansion); the hardcoded generic-name wordlist is only used when BUCKET_SCAN_WORDLIST=true.
This commit is contained in:
StasonJatham committed
1 parent
a899362e18
commit
ebb020c6b7
4 files changed
+53
-45
No files matched your search
@@ -35,3 +35,4 @@ BUCKET_SCAN_QPS=50
|
||||
BUCKET_SCAN_WORKERS=16
|
||||
BUCKET_SCAN_QUEUE=65536
|
||||
BUCKET_SCAN_DO=false
|
||||
BUCKET_SCAN_WORDLIST=false
|
||||
@@ -132,6 +132,7 @@ Environment variables only; no config files, no runtime pattern files.
|
||||
| `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers |
|
||||
| `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog |
|
||||
| `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions |
|
||||
| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names |
|
||||
| `DEDUP_TTL` | `1h` | hostname/finding dedup window |
|
||||
| `METRICS_ADDR` | `:9090` | metrics/health/UI listener |
|
||||
| `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI |
|
||||
|
||||
+7
-3
@@ -167,12 +167,16 @@ func run() error {
|
||||
},
|
||||
})
|
||||
discoverer.Start(ctx)
|
||||
for _, name := range discover.Wordlist() {
|
||||
discoverer.SubmitName(name)
|
||||
wordlist := 0
|
||||
if cfg.BucketScanWordlist {
|
||||
for _, name := range discover.Wordlist() {
|
||||
discoverer.SubmitName(name)
|
||||
}
|
||||
wordlist = len(discover.Wordlist())
|
||||
}
|
||||
slog.Info("bucket name discovery enabled", "event", "startup",
|
||||
"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO,
|
||||
"wordlist", len(discover.Wordlist()))
|
||||
"wordlist", wordlist)
|
||||
}
|
||||
|
||||
var prober *probe.Prober
|
||||
|
||||
+44
-42
@@ -39,11 +39,12 @@ type Config struct {
|
||||
IndexEnabled bool
|
||||
IndexMaxObjects int
|
||||
|
||||
BucketScanEnabled bool
|
||||
BucketScanQPS int
|
||||
BucketScanWorkers int
|
||||
BucketScanQueue int
|
||||
BucketScanDO bool
|
||||
BucketScanEnabled bool
|
||||
BucketScanQPS int
|
||||
BucketScanWorkers int
|
||||
BucketScanQueue int
|
||||
BucketScanDO bool
|
||||
BucketScanWordlist bool
|
||||
|
||||
MetricsAddr string
|
||||
UIUser string
|
||||
@@ -63,43 +64,44 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
|
||||
|
||||
func Load() (*Config, error) {
|
||||
c := &Config{
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
DBMaxConns: envInt("DB_MAX_CONNS", 8),
|
||||
DNSServers: envList("DNS_SERVERS", nil),
|
||||
DNSWorkers: envInt("DNS_WORKERS", 64),
|
||||
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
|
||||
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
|
||||
DedupSize: envInt("DEDUP_SIZE", 1<<18),
|
||||
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
|
||||
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
|
||||
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
|
||||
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
|
||||
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
|
||||
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
|
||||
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
|
||||
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
|
||||
ActiveProbing: envBool("ACTIVE_PROBING", false),
|
||||
ProbeWorkers: envInt("PROBE_WORKERS", 8),
|
||||
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
|
||||
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
|
||||
ProbeDeep: envBool("PROBE_DEEP", true),
|
||||
IndexEnabled: envBool("INDEX_ENABLED", true),
|
||||
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
|
||||
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
|
||||
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
|
||||
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
|
||||
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
|
||||
BucketScanDO: envBool("BUCKET_SCAN_DO", false),
|
||||
MetricsAddr: envString("METRICS_ADDR", ":9090"),
|
||||
UIUser: envString("UI_USER", ""),
|
||||
UIPass: envString("UI_PASS", ""),
|
||||
LogLevel: envString("LOG_LEVEL", "info"),
|
||||
LogFormat: envString("LOG_FORMAT", "text"),
|
||||
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
|
||||
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
|
||||
StartAtEnd: envBool("START_AT_END", false),
|
||||
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
|
||||
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
DBMaxConns: envInt("DB_MAX_CONNS", 8),
|
||||
DNSServers: envList("DNS_SERVERS", nil),
|
||||
DNSWorkers: envInt("DNS_WORKERS", 64),
|
||||
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
|
||||
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
|
||||
DedupSize: envInt("DEDUP_SIZE", 1<<18),
|
||||
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
|
||||
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
|
||||
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
|
||||
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
|
||||
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
|
||||
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
|
||||
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
|
||||
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
|
||||
ActiveProbing: envBool("ACTIVE_PROBING", false),
|
||||
ProbeWorkers: envInt("PROBE_WORKERS", 8),
|
||||
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
|
||||
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
|
||||
ProbeDeep: envBool("PROBE_DEEP", true),
|
||||
IndexEnabled: envBool("INDEX_ENABLED", true),
|
||||
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
|
||||
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
|
||||
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
|
||||
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
|
||||
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
|
||||
BucketScanDO: envBool("BUCKET_SCAN_DO", false),
|
||||
BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false),
|
||||
MetricsAddr: envString("METRICS_ADDR", ":9090"),
|
||||
UIUser: envString("UI_USER", ""),
|
||||
UIPass: envString("UI_PASS", ""),
|
||||
LogLevel: envString("LOG_LEVEL", "info"),
|
||||
LogFormat: envString("LOG_FORMAT", "text"),
|
||||
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
|
||||
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
|
||||
StartAtEnd: envBool("START_AT_END", false),
|
||||
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
|
||||
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
|
||||
}
|
||||
if err := c.validate(); err != nil {
|
||||
return nil, err
|
||||
|
||||
Reference in new issue
Block a user