StasonJatham
a899362e18
CMS SANs are custom domains, not bucket names, so matching SANs finds almost no buckets. This adds internal/discover: from each non-matching hostname it derives candidate names (domain label + subdomain label + ~40 suffixes) plus a built-in wordlist, and anonymously tests listing on S3, GCS and (optionally) DigitalOcean Spaces. Every 200 is indexed (keys/sizes/dates only). Bounded hard: global BUCKET_SCAN_QPS rate limit, 12h per-name dedup, random sampled domain queue, MaxPerDomain cap. Never downloads contents. Fix: GCS candidates now use the virtual-host form (<bucket>.storage.googleapis.com) so buckets no longer collide on the cloud_assets unique key; previously they all shared storage.googleapis.com and overwrote each other. Live: 30 open buckets in ~5 min (e.g. estimator-dev, image-prod, update-media, mos-images S3; several GCS buckets), 11k+ objects indexed.
64 lines
1.6 KiB
Go
64 lines
1.6 KiB
Go
package discover
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestCandidates(t *testing.T) {
|
|
got := candidates("assets.example.com", 200)
|
|
set := map[string]bool{}
|
|
for _, c := range got {
|
|
set[c] = true
|
|
}
|
|
for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} {
|
|
if !set[want] {
|
|
t.Fatalf("candidates missing %q (got %v)", want, got)
|
|
}
|
|
}
|
|
for _, c := range got {
|
|
if !validBucketName(c) {
|
|
t.Fatalf("invalid candidate %q", c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBaseLabelsSkipsNoise(t *testing.T) {
|
|
// www is skipped, so only the registrable label remains.
|
|
got := baseLabels("www.example.com")
|
|
if len(got) != 1 || got[0] != "example" {
|
|
t.Fatalf("baseLabels = %v, want [example]", got)
|
|
}
|
|
if baseLabels("example") != nil {
|
|
t.Fatal("single label must yield no base")
|
|
}
|
|
}
|
|
|
|
func TestTargets(t *testing.T) {
|
|
d := New(Config{S3: true, GCS: true, DOSpaces: true, DORegions: []string{"nyc3"}})
|
|
ts := d.targets("example")
|
|
if len(ts) != 3 {
|
|
t.Fatalf("got %d targets, want 3", len(ts))
|
|
}
|
|
var s3, gcs, do bool
|
|
for _, tg := range ts {
|
|
switch {
|
|
case strings.Contains(tg.URL, "example.s3.amazonaws.com/?list-type=2"):
|
|
s3 = true
|
|
case strings.Contains(tg.URL, "storage.googleapis.com/example"):
|
|
gcs = true
|
|
case strings.Contains(tg.URL, "example.nyc3.digitaloceanspaces.com/?list-type=2"):
|
|
do = true
|
|
}
|
|
}
|
|
if !s3 || !gcs || !do {
|
|
t.Fatalf("missing targets: s3=%v gcs=%v do=%v (%v)", s3, gcs, do, ts)
|
|
}
|
|
}
|
|
|
|
func TestMaxPerDomain(t *testing.T) {
|
|
if got := candidates("example.com", 5); len(got) != 5 {
|
|
t.Fatalf("MaxPerDomain not honored: got %d", len(got))
|
|
}
|
|
}
|