karl/bucketeer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bucket discovery: no name guessing

Browse Source 
Remove all fabricated candidate names (suffixes and the wordlist). Discovery now
only tests names that actually occur in the incoming hostname: the subdomain
label and the registrable-domain label (assets.example.com -> assets, example).
Nothing is permuted or drawn from a wordlist.
This commit is contained in:
StasonJatham committed 2026-09-22 16:09:44 +02:00
1 parent ebb020c6b7
commit fe5c01666b
6 files changed
+56 -121
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.env.example
-1
View File
@@ -35,4 +35,3 @@ BUCKET_SCAN_QPS=50
BUCKET_SCAN_WORKERS=16
BUCKET_SCAN_QUEUE=65536
BUCKET_SCAN_DO=false
BUCKET_SCAN_WORDLIST=false
README.md
+3 -4
View File
@@ -132,7 +132,6 @@ Environment variables only; no config files, no runtime pattern files.
| `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers |
| `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog |
| `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions |
| `BUCKET_SCAN_WORDLIST` | `false` | also test a built-in list of common generic names |
| `DEDUP_TTL` | `1h` | hostname/finding dedup window |
| `METRICS_ADDR` | `:9090` | metrics/health/UI listener |
| `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI |
@@ -317,9 +316,9 @@ Certificate SANs are custom domains, not bucket names, so matching SANs alone
finds almost no buckets. Instead, every non-matching hostname feeds a discovery
engine that:
1. derives candidate names from the domain (`example` → `example`,
`example-assets`, `example-backups`, …, plus the subdomain label), also
testing a built-in wordlist of common names;
1. takes the names that actually occur in the hostname — the subdomain label
and the registrable-domain label (`assets.example.com` → `assets`,
`example`). No names are fabricated, permuted, or drawn from a wordlist;
2. tests anonymous listing per provider —
`https://<name>.s3.amazonaws.com/?list-type=2` (AWS),
`https://<name>.storage.googleapis.com` (GCS),
cmd/ctcloud/main.go
+1 -9
View File
@@ -167,16 +167,8 @@ func run() error {
},
})
discoverer.Start(ctx)
wordlist := 0
if cfg.BucketScanWordlist {
for _, name := range discover.Wordlist() {
discoverer.SubmitName(name)
}
wordlist = len(discover.Wordlist())
}
slog.Info("bucket name discovery enabled", "event", "startup",
"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO,
"wordlist", wordlist)
"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO)
}
var prober *probe.Prober
internal/config/config.go
+42 -44
View File
@@ -39,12 +39,11 @@ type Config struct {
IndexEnabled bool
IndexMaxObjects int
BucketScanEnabled bool
BucketScanQPS int
BucketScanWorkers int
BucketScanQueue int
BucketScanDO bool
BucketScanWordlist bool
BucketScanEnabled bool
BucketScanQPS int
BucketScanWorkers int
BucketScanQueue int
BucketScanDO bool
MetricsAddr string
UIUser string
@@ -64,44 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
func Load() (*Config, error) {
c := &Config{
DatabaseURL: os.Getenv("DATABASE_URL"),
DBMaxConns: envInt("DB_MAX_CONNS", 8),
DNSServers: envList("DNS_SERVERS", nil),
DNSWorkers: envInt("DNS_WORKERS", 64),
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
DedupSize: envInt("DEDUP_SIZE", 1<<18),
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
ActiveProbing: envBool("ACTIVE_PROBING", false),
ProbeWorkers: envInt("PROBE_WORKERS", 8),
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
ProbeDeep: envBool("PROBE_DEEP", true),
IndexEnabled: envBool("INDEX_ENABLED", true),
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
BucketScanDO: envBool("BUCKET_SCAN_DO", false),
BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false),
MetricsAddr: envString("METRICS_ADDR", ":9090"),
UIUser: envString("UI_USER", ""),
UIPass: envString("UI_PASS", ""),
LogLevel: envString("LOG_LEVEL", "info"),
LogFormat: envString("LOG_FORMAT", "text"),
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
StartAtEnd: envBool("START_AT_END", false),
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
DatabaseURL: os.Getenv("DATABASE_URL"),
DBMaxConns: envInt("DB_MAX_CONNS", 8),
DNSServers: envList("DNS_SERVERS", nil),
DNSWorkers: envInt("DNS_WORKERS", 64),
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
DedupSize: envInt("DEDUP_SIZE", 1<<18),
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144),
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
ActiveProbing: envBool("ACTIVE_PROBING", false),
ProbeWorkers: envInt("PROBE_WORKERS", 8),
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
ProbeDeep: envBool("PROBE_DEEP", true),
IndexEnabled: envBool("INDEX_ENABLED", true),
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
BucketScanQPS: envInt("BUCKET_SCAN_QPS", 50),
BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
BucketScanQueue: envInt("BUCKET_SCAN_QUEUE", 65536),
BucketScanDO: envBool("BUCKET_SCAN_DO", false),
MetricsAddr: envString("METRICS_ADDR", ":9090"),
UIUser: envString("UI_USER", ""),
UIPass: envString("UI_PASS", ""),
LogLevel: envString("LOG_LEVEL", "info"),
LogFormat: envString("LOG_FORMAT", "text"),
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
StartAtEnd: envBool("START_AT_END", false),
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
}
if err := c.validate(); err != nil {
return nil, err
internal/discover/discover.go
+6 -52
View File
@@ -45,7 +45,6 @@ type Discoverer struct {
cfg Config
client *http.Client
q *domainQueue
names *domainQueue
tokens chan struct{}
seen *dns.TTLCache[bool] // candidate names already tested
stop chan struct{}
@@ -69,7 +68,6 @@ func New(cfg Config) *Discoverer {
cfg: cfg,
client: &http.Client{Timeout: 6 * time.Second},
q: newDomainQueue(cfg.Queue),
names: newDomainQueue(cfg.Queue),
tokens: make(chan struct{}, max(cfg.QPS, 1)),
seen: dns.NewTTLCache[bool](1<<20, 64),
stop: make(chan struct{}),
@@ -138,7 +136,6 @@ func (d *Discoverer) worker(ctx context.Context) {
case <-d.stop:
return
case <-d.q.notify:
case <-d.names.notify:
}
for {
host, ok := d.q.pop()
@@ -147,13 +144,6 @@ func (d *Discoverer) worker(ctx context.Context) {
}
d.scanDomain(ctx, host)
}
for {
name, ok := d.names.pop()
if !ok {
break
}
d.testName(ctx, name)
}
}
}
@@ -166,13 +156,6 @@ func (d *Discoverer) scanDomain(ctx context.Context, host string) {
}
}
// SubmitName tests a specific candidate bucket name (e.g. from a wordlist).
func (d *Discoverer) SubmitName(name string) {
if validBucketName(name) {
d.names.push(name)
}
}
func (d *Discoverer) testName(ctx context.Context, name string) {
if _, dup := d.seen.Get(name); dup {
return
@@ -272,30 +255,9 @@ func (d *Discoverer) test(ctx context.Context, t Target) {
// --- candidate generation ---
var suffixes = []string{
"", "-assets", "-static", "-backups", "-backup", "-images", "-img", "-media",
"-prod", "-dev", "-staging", "-test", "-files", "-uploads", "-public", "-cdn",
"-data", "-content", "-site", "-web", "-www", "-app", "-store", "-downloads",
"-photos", "-videos", "-docs", "-documents", "-archive", "-logs", "-artifacts",
"-build", "-deploy", "-release", "-db", "-database", "-sql", "-exports", "-reports",
"-assets-prod", "-assets-dev", "-static-prod", "-uploads-prod",
}
// Wordlist is a bounded set of common global bucket names tested once each.
func Wordlist() []string {
return []string{
"backups", "backup", "images", "image", "img", "media", "static", "assets",
"uploads", "files", "file", "data", "content", "public", "cdn", "www", "web",
"website", "app", "store", "downloads", "docs", "documents", "archive",
"logs", "artifacts", "build", "deploy", "release", "db", "database", "sql",
"dump", "exports", "reports", "photos", "videos", "dev", "test", "staging",
"prod", "production", "internal", "private", "secure", "userdata", "usercontent",
"thumbnails", "avatars", "profiles", "attachments", "tmp", "temp", "cache",
"config", "secrets", "keys", "certificates", "certs", "gifs", "png", "jpeg",
"download", "uploads-test", "uploads-dev", "assets-test", "assets-dev",
"static-assets", "web-assets", "app-assets", "site-assets", "media-assets",
}
}
// No name guessing: candidates are exactly the labels that occur in the incoming
// hostname (the subdomain label and the registrable-domain label). Nothing is
// fabricated, permuted, or drawn from a wordlist.
// baseLabels returns plausible bucket-name stems from a hostname: the leftmost
// label and the registrable-domain label.
@@ -325,17 +287,9 @@ var skipLabel = map[string]bool{
}
func candidates(host string, max int) []string {
var out []string
for _, b := range baseLabels(host) {
for _, s := range suffixes {
name := b + s
if validBucketName(name) {
out = append(out, name)
}
if max > 0 && len(out) >= max {
return out
}
}
out := baseLabels(host)
if max > 0 && len(out) > max {
out = out[:max]
}
return out
}
internal/discover/discover_test.go
+4 -11
View File
@@ -5,16 +5,10 @@ import (
"testing"
)
func TestCandidates(t *testing.T) {
func TestCandidatesAreExactLabelsOnly(t *testing.T) {
got := candidates("assets.example.com", 200)
set := map[string]bool{}
for _, c := range got {
set[c] = true
}
for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} {
if !set[want] {
t.Fatalf("candidates missing %q (got %v)", want, got)
}
if len(got) != 2 || got[0] != "assets" || got[1] != "example" {
t.Fatalf("candidates = %v, want [assets example] (no fabricated names)", got)
}
for _, c := range got {
if !validBucketName(c) {
@@ -24,7 +18,6 @@ func TestCandidates(t *testing.T) {
}
func TestBaseLabelsSkipsNoise(t *testing.T) {
// www is skipped, so only the registrable label remains.
got := baseLabels("www.example.com")
if len(got) != 1 || got[0] != "example" {
t.Fatalf("baseLabels = %v, want [example]", got)
@@ -57,7 +50,7 @@ func TestTargets(t *testing.T) {
}
func TestMaxPerDomain(t *testing.T) {
if got := candidates("example.com", 5); len(got) != 5 {
if got := candidates("example.com", 5); len(got) > 5 {
t.Fatalf("MaxPerDomain not honored: got %d", len(got))
}
}