karl/bucketeer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bucket discovery: no name guessing

Browse Source 
Remove all fabricated candidate names (suffixes and the wordlist). Discovery now
only tests names that actually occur in the incoming hostname: the subdomain
label and the registrable-domain label (assets.example.com -> assets, example).
Nothing is permuted or drawn from a wordlist.
This commit is contained in:
StasonJatham committed 2026-09-22 16:09:44 +02:00
1 parent ebb020c6b7
commit fe5c01666b
6 files changed
+56 -121
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.env.example
-1
View File
@@ -35,4 +35,3 @@ BUCKET_SCAN_QPS=50
BUCKET_SCAN_WORKERS=16 BUCKET_SCAN_WORKERS=16
BUCKET_SCAN_QUEUE=65536 BUCKET_SCAN_QUEUE=65536
BUCKET_SCAN_DO=false BUCKET_SCAN_DO=false
BUCKET_SCAN_WORDLIST=false
README.md
+3 -4
View File
@@ -132,7 +132,6 @@ Environment variables only; no config files, no runtime pattern files.
| `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers | | `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers |
| `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog | | `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog |
| `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions | | `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions |
| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names |
| `DEDUP_TTL` | `1h` | hostname/finding dedup window | | `DEDUP_TTL` | `1h` | hostname/finding dedup window |
| `METRICS_ADDR` | `:9090` | metrics/health/UI listener | | `METRICS_ADDR` | `:9090` | metrics/health/UI listener |
| `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI | | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI |
@@ -317,9 +316,9 @@ Certificate SANs are custom domains, not bucket names, so matching SANs alone
finds almost no buckets. Instead, every non-matching hostname feeds a discovery finds almost no buckets. Instead, every non-matching hostname feeds a discovery
engine that: engine that:
1. derives candidate names from the domain (`example` → `example`, 1. takes the names that actually occur in the hostname — the subdomain label
`example-assets`, `example-backups`, …, plus the subdomain label), also and the registrable-domain label (`assets.example.com` → `assets`,
testing a built-in wordlist of common names; `example`). No names are fabricated, permuted, or drawn from a wordlist;
2. tests anonymous listing per provider — 2. tests anonymous listing per provider —
`https://<name>.s3.amazonaws.com/?list-type=2` (AWS), `https://<name>.s3.amazonaws.com/?list-type=2` (AWS),
`https://<name>.storage.googleapis.com` (GCS), `https://<name>.storage.googleapis.com` (GCS),
cmd/ctcloud/main.go
+1 -9
View File
@@ -167,16 +167,8 @@ func run() error {
}, },
}) })
discoverer.Start(ctx) discoverer.Start(ctx)
wordlist := 0
if cfg.BucketScanWordlist {
for _, name := range discover.Wordlist() {
discoverer.SubmitName(name)
}
wordlist = len(discover.Wordlist())
}
slog.Info("bucket name discovery enabled", "event", "startup", slog.Info("bucket name discovery enabled", "event", "startup",
"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO, "qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO)
"wordlist", wordlist)
} }
var prober *probe.Prober var prober *probe.Prober
internal/config/config.go
+42 -44
View File
@@ -39,12 +39,11 @@ type Config struct {
IndexEnabled bool IndexEnabled bool
IndexMaxObjects int IndexMaxObjects int
BucketScanEnabled bool BucketScanEnabled bool
BucketScanQPS int BucketScanQPS int
BucketScanWorkers int BucketScanWorkers int
BucketScanQueue int BucketScanQueue int
BucketScanDO bool BucketScanDO bool
BucketScanWordlist bool
MetricsAddr string MetricsAddr string
UIUser string UIUser string
@@ -64,44 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
func Load() (*Config, error) { func Load() (*Config, error) {
c := &Config{ c := &Config{
DatabaseURL: os.Getenv("DATABASE_URL"), DatabaseURL: os.Getenv("DATABASE_URL"),
DBMaxConns: envInt("DB_MAX_CONNS", 8), DBMaxConns: envInt("DB_MAX_CONNS", 8),
DNSServers: envList("DNS_SERVERS", nil), DNSServers: envList("DNS_SERVERS", nil),
DNSWorkers: envInt("DNS_WORKERS", 64), DNSWorkers: envInt("DNS_WORKERS", 64),
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
DedupSize: envInt("DEDUP_SIZE", 1<<18), DedupSize: envInt("DEDUP_SIZE", 1<<18),
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
DBBatchSize: envInt("DB_BATCH_SIZE", 500), DBBatchSize: envInt("DB_BATCH_SIZE", 500),
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
ActiveProbing: envBool("ACTIVE_PROBING", false), ActiveProbing: envBool("ACTIVE_PROBING", false),
ProbeWorkers: envInt("PROBE_WORKERS", 8), ProbeWorkers: envInt("PROBE_WORKERS", 8),
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16), ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
ProbeDeep: envBool("PROBE_DEEP", true), ProbeDeep: envBool("PROBE_DEEP", true),
IndexEnabled: envBool("INDEX_ENABLED", true), IndexEnabled: envBool("INDEX_ENABLED", true),
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000), IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true), BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50), BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16), BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536), BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
BucketScanDO: envBool("BUCKET_SCAN_DO", false), BucketScanDO: envBool("BUCKET_SCAN_DO", false),
BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false), MetricsAddr: envString("METRICS_ADDR", ":9090"),
MetricsAddr: envString("METRICS_ADDR", ":9090"), UIUser: envString("UI_USER", ""),
UIUser: envString("UI_USER", ""), UIPass: envString("UI_PASS", ""),
UIPass: envString("UI_PASS", ""), LogLevel: envString("LOG_LEVEL", "info"),
LogLevel: envString("LOG_LEVEL", "info"), LogFormat: envString("LOG_FORMAT", "text"),
LogFormat: envString("LOG_FORMAT", "text"), StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), StartAtEnd: envBool("START_AT_END", false),
StartAtEnd: envBool("START_AT_END", false), ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
} }
if err := c.validate(); err != nil { if err := c.validate(); err != nil {
return nil, err return nil, err
internal/discover/discover.go
+6 -52
View File
@@ -45,7 +45,6 @@ type Discoverer struct {
cfg Config cfg Config
client *http.Client client *http.Client
q *domainQueue q *domainQueue
names *domainQueue
tokens chan struct{} tokens chan struct{}
seen *dns.TTLCache[bool] // candidate names already tested seen *dns.TTLCache[bool] // candidate names already tested
stop chan struct{} stop chan struct{}
@@ -69,7 +68,6 @@ func New(cfg Config) *Discoverer {
cfg: cfg, cfg: cfg,
client: &http.Client{Timeout: 6 * time.Second}, client: &http.Client{Timeout: 6 * time.Second},
q: newDomainQueue(cfg.Queue), q: newDomainQueue(cfg.Queue),
names: newDomainQueue(cfg.Queue),
tokens: make(chan struct{}, max(cfg.QPS, 1)), tokens: make(chan struct{}, max(cfg.QPS, 1)),
seen: dns.NewTTLCache[bool](1<<20, 64), seen: dns.NewTTLCache[bool](1<<20, 64),
stop: make(chan struct{}), stop: make(chan struct{}),
@@ -138,7 +136,6 @@ func (d *Discoverer) worker(ctx context.Context) {
case <-d.stop: case <-d.stop:
return return
case <-d.q.notify: case <-d.q.notify:
case <-d.names.notify:
} }
for { for {
host, ok := d.q.pop() host, ok := d.q.pop()
@@ -147,13 +144,6 @@ func (d *Discoverer) worker(ctx context.Context) {
} }
d.scanDomain(ctx, host) d.scanDomain(ctx, host)
} }
for {
name, ok := d.names.pop()
if !ok {
break
}
d.testName(ctx, name)
}
} }
} }
@@ -166,13 +156,6 @@ func (d *Discoverer) scanDomain(ctx context.Context, host string) {
} }
} }
// SubmitName tests a specific candidate bucket name (e.g. from a wordlist).
func (d *Discoverer) SubmitName(name string) {
if validBucketName(name) {
d.names.push(name)
}
}
func (d *Discoverer) testName(ctx context.Context, name string) { func (d *Discoverer) testName(ctx context.Context, name string) {
if _, dup := d.seen.Get(name); dup { if _, dup := d.seen.Get(name); dup {
return return
@@ -272,30 +255,9 @@ func (d *Discoverer) test(ctx context.Context, t Target) {
// --- candidate generation --- // --- candidate generation ---
var suffixes = []string{ // No name guessing: candidates are exactly the labels that occur in the incoming
"", "-assets", "-static", "-backups", "-backup", "-images", "-img", "-media", // hostname (the subdomain label and the registrable-domain label). Nothing is
"-prod", "-dev", "-staging", "-test", "-files", "-uploads", "-public", "-cdn", // fabricated, permuted, or drawn from a wordlist.
"-data", "-content", "-site", "-web", "-www", "-app", "-store", "-downloads",
"-photos", "-videos", "-docs", "-documents", "-archive", "-logs", "-artifacts",
"-build", "-deploy", "-release", "-db", "-database", "-sql", "-exports", "-reports",
"-assets-prod", "-assets-dev", "-static-prod", "-uploads-prod",
}
// Wordlist is a bounded set of common global bucket names tested once each.
func Wordlist() []string {
return []string{
"backups", "backup", "images", "image", "img", "media", "static", "assets",
"uploads", "files", "file", "data", "content", "public", "cdn", "www", "web",
"website", "app", "store", "downloads", "docs", "documents", "archive",
"logs", "artifacts", "build", "deploy", "release", "db", "database", "sql",
"dump", "exports", "reports", "photos", "videos", "dev", "test", "staging",
"prod", "production", "internal", "private", "secure", "userdata", "usercontent",
"thumbnails", "avatars", "profiles", "attachments", "tmp", "temp", "cache",
"config", "secrets", "keys", "certificates", "certs", "gifs", "png", "jpeg",
"download", "uploads-test", "uploads-dev", "assets-test", "assets-dev",
"static-assets", "web-assets", "app-assets", "site-assets", "media-assets",
}
}
// baseLabels returns plausible bucket-name stems from a hostname: the leftmost // baseLabels returns plausible bucket-name stems from a hostname: the leftmost
// label and the registrable-domain label. // label and the registrable-domain label.
@@ -325,17 +287,9 @@ var skipLabel = map[string]bool{
} }
func candidates(host string, max int) []string { func candidates(host string, max int) []string {
var out []string out := baseLabels(host)
for _, b := range baseLabels(host) { if max > 0 && len(out) > max {
for _, s := range suffixes { out = out[:max]
name := b + s
if validBucketName(name) {
out = append(out, name)
}
if max > 0 && len(out) >= max {
return out
}
}
} }
return out return out
} }
internal/discover/discover_test.go
+4 -11
View File
@@ -5,16 +5,10 @@ import (
"testing" "testing"
) )
func TestCandidates(t *testing.T) { func TestCandidatesAreExactLabelsOnly(t *testing.T) {
got := candidates("assets.example.com", 200) got := candidates("assets.example.com", 200)
set := map[string]bool{} if len(got) != 2 || got[0] != "assets" || got[1] != "example" {
for _, c := range got { t.Fatalf("candidates = %v, want [assets example] (no fabricated names)", got)
set[c] = true
}
for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} {
if !set[want] {
t.Fatalf("candidates missing %q (got %v)", want, got)
}
} }
for _, c := range got { for _, c := range got {
if !validBucketName(c) { if !validBucketName(c) {
@@ -24,7 +18,6 @@ func TestCandidates(t *testing.T) {
} }
func TestBaseLabelsSkipsNoise(t *testing.T) { func TestBaseLabelsSkipsNoise(t *testing.T) {
// www is skipped, so only the registrable label remains.
got := baseLabels("www.example.com") got := baseLabels("www.example.com")
if len(got) != 1 || got[0] != "example" { if len(got) != 1 || got[0] != "example" {
t.Fatalf("baseLabels = %v, want [example]", got) t.Fatalf("baseLabels = %v, want [example]", got)
@@ -57,7 +50,7 @@ func TestTargets(t *testing.T) {
} }
func TestMaxPerDomain(t *testing.T) { func TestMaxPerDomain(t *testing.T) {
if got := candidates("example.com", 5); len(got) != 5 { if got := candidates("example.com", 5); len(got) > 5 {
t.Fatalf("MaxPerDomain not honored: got %d", len(got)) t.Fatalf("MaxPerDomain not honored: got %d", len(got))
} }
} }