discover Azure Blob containers by account name

Azure has two levels: the account maps to DNS (<account>.blob.core.windows.net)
and anonymous access is per container with no anonymous container enumeration.
Discovery now tests incoming labels as Azure account names (DNS existence) and,
for accounts that exist, probes a small set of common containers, indexing any
that list anonymously. CONTENT still never downloaded.

Also document that docker compose down -v wipes all discovered assets; use
stop/start to keep the pgdata volume.

Live: connectsystem.blob.core.windows.net indexed (33 objects).
This commit is contained in:
StasonJatham committed 2026-09-22 17:28:41 +02:00
1 parent dc7bdae99d
commit 32035d0b9d
3 files changed
+111 -4

No files matched your search

+86
View File
@@ -160,6 +160,92 @@ func (d *Discoverer) testName(ctx context.Context, name string) {
}
d.test(ctx, t)
}
d.scanAzure(ctx, name)
}
// scanAzure handles Azure's two-level model. The account name maps to DNS
// (<account>.blob.core.windows.net), so a nonexistent account fails the first
// request and we stop. Anonymous access is per container and Microsoft offers no
// anonymous container enumeration, so the container names are the one place a
// wordlist is unavoidable.
func (d *Discoverer) scanAzure(ctx context.Context, account string) {
if !validAzureAccount(account) {
return
}
key := "az:" + account
if _, dup := d.seen.Get(key); dup {
return
}
d.seen.Set(key, true, 12*time.Hour)
host := account + ".blob.core.windows.net"
if !d.acquire(ctx) {
return
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/?comp=list", nil)
if err != nil {
return
}
resp, err := d.client.Do(req)
if err != nil {
return // account does not exist (NXDOMAIN) or is unreachable
}
_, _ = io.CopyN(io.Discard, resp.Body, 512)
resp.Body.Close()
// Account exists. Try known/common containers for anonymous listing.
var (
entries []model.Entry
found []string
)
for _, container := range index.AzureContainers(account) {
if ctx.Err() != nil {
return
}
if d.cfg.Metrics != nil {
d.cfg.Metrics.BucketCandidates.Add(1)
}
if !d.acquire(ctx) {
return
}
c := model.Candidate{
Hostname: host, Original: host, Resource: account, Container: container,
Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
ResourceSpecific: true, Source: model.SourceDiscovery,
}
e, err := d.cfg.Indexer.List(ctx, c)
if err != nil || len(e) == 0 {
continue
}
entries = append(entries, e...)
found = append(found, container)
}
if len(entries) == 0 || d.cfg.OnOpen == nil {
return
}
if d.cfg.Metrics != nil {
d.cfg.Metrics.BucketOpen.Add(1)
}
c := model.Candidate{
Hostname: host, Original: host, Resource: account,
Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
ResourceSpecific: true, Source: model.SourceDiscovery,
}
d.cfg.OnOpen(ctx, c, "azure public container(s): "+strings.Join(found, ","), entries)
}
// validAzureAccount mirrors Azure's naming rules: 3-24 lowercase alphanumerics.
func validAzureAccount(s string) bool {
if len(s) < 3 || len(s) > 24 {
return false
}
for i := 0; i < len(s); i++ {
c := s[i]
if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9') {
return false
}
}
return true
}
// Target is one provider listing endpoint for a candidate name.
+13
View File
@@ -17,6 +17,19 @@ func TestCandidatesAreExactLabelsOnly(t *testing.T) {
}
}
func TestValidAzureAccount(t *testing.T) {
for _, ok := range []string{"myaccount", "abc123", "a1b2c3d4e5f6g7h8i9j0k1l2"} {
if !validAzureAccount(ok) {
t.Fatalf("%q should be a valid Azure account name", ok)
}
}
for _, bad := range []string{"ab", "has-hyphen", "UPPER", "with.dot", "a1b2c3d4e5f6g7h8i9j0k1l2m3"} {
if validAzureAccount(bad) {
t.Fatalf("%q should be rejected as an Azure account name", bad)
}
}
}
func TestBaseLabelsSkipsNoise(t *testing.T) {
got := baseLabels("www.example.com")
if len(got) != 1 || got[0] != "example" {