From 32035d0b9d1a20d4a2015796e075e266a22e5636 Mon Sep 17 00:00:00 2001 From: StasonJatham Date: Tue, 22 Sep 2026 17:28:41 +0200 Subject: [PATCH] discover Azure Blob containers by account name Azure has two levels: the account maps to DNS (.blob.core.windows.net) and anonymous access is per container with no anonymous container enumeration. Discovery now tests incoming labels as Azure account names (DNS existence) and, for accounts that exist, probes a small set of common containers, indexing any that list anonymously. CONTENT still never downloaded. Also document that docker compose down -v wipes all discovered assets; use stop/start to keep the pgdata volume. Live: connectsystem.blob.core.windows.net indexed (33 objects). --- README.md | 16 ++++-- internal/discover/discover.go | 86 ++++++++++++++++++++++++++++++ internal/discover/discover_test.go | 13 +++++ 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index efdac84..a0d5af0 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,11 @@ docker compose up --build That starts `postgres` and the stateless `ctcloud` container. The application container runs as non-root, has a read-only root filesystem, drops all Linux capabilities, sets CPU/memory/pids limits and mounts no application volume. The -only persistent state is PostgreSQL (CT checkpoints + cloud assets). +only persistent state is PostgreSQL (CT checkpoints + cloud assets + index). + +> **Findings accumulate over time.** Use `docker compose stop` / `docker compose +> start` (or plain `up -d`) to keep the `pgdata` volume. `docker compose down -v` +> deletes the volume and every discovered asset with it. Endpoints on port `9090`, bound to `127.0.0.1` only: @@ -324,9 +328,13 @@ engine that: 3. indexes every bucket that returns 200 (keys/sizes/dates only). It is aggressively bounded: a global `BUCKET_SCAN_QPS` limit, a 12h per-name -dedup cache, and a random-sampled domain queue. Azure cannot be enumerated this -way (account names are the namespace and containers need names), so Azure is -handled from CT accounts plus a container wordlist, and R2 is never listable. +dedup cache, and a random-sampled domain queue. + +Azure is handled separately because it has two levels: the account name maps to +DNS (`.blob.core.windows.net`), and anonymous access is per container +with no anonymous container enumeration — so for accounts that exist we probe a +small set of common container names. R2 is never listable and is only probed for +objects. This is what actually yields open buckets. Of the buckets found, S3 and GCS return full object listings; Azure needs a known container; R2 only serves diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 173ada2..19563fc 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -160,6 +160,92 @@ func (d *Discoverer) testName(ctx context.Context, name string) { } d.test(ctx, t) } + d.scanAzure(ctx, name) +} + +// scanAzure handles Azure's two-level model. The account name maps to DNS +// (.blob.core.windows.net), so a nonexistent account fails the first +// request and we stop. Anonymous access is per container and Microsoft offers no +// anonymous container enumeration, so the container names are the one place a +// wordlist is unavoidable. +func (d *Discoverer) scanAzure(ctx context.Context, account string) { + if !validAzureAccount(account) { + return + } + key := "az:" + account + if _, dup := d.seen.Get(key); dup { + return + } + d.seen.Set(key, true, 12*time.Hour) + + host := account + ".blob.core.windows.net" + if !d.acquire(ctx) { + return + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/?comp=list", nil) + if err != nil { + return + } + resp, err := d.client.Do(req) + if err != nil { + return // account does not exist (NXDOMAIN) or is unreachable + } + _, _ = io.CopyN(io.Discard, resp.Body, 512) + resp.Body.Close() + + // Account exists. Try known/common containers for anonymous listing. + var ( + entries []model.Entry + found []string + ) + for _, container := range index.AzureContainers(account) { + if ctx.Err() != nil { + return + } + if d.cfg.Metrics != nil { + d.cfg.Metrics.BucketCandidates.Add(1) + } + if !d.acquire(ctx) { + return + } + c := model.Candidate{ + Hostname: host, Original: host, Resource: account, Container: container, + Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, + ResourceSpecific: true, Source: model.SourceDiscovery, + } + e, err := d.cfg.Indexer.List(ctx, c) + if err != nil || len(e) == 0 { + continue + } + entries = append(entries, e...) + found = append(found, container) + } + if len(entries) == 0 || d.cfg.OnOpen == nil { + return + } + if d.cfg.Metrics != nil { + d.cfg.Metrics.BucketOpen.Add(1) + } + c := model.Candidate{ + Hostname: host, Original: host, Resource: account, + Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, + ResourceSpecific: true, Source: model.SourceDiscovery, + } + d.cfg.OnOpen(ctx, c, "azure public container(s): "+strings.Join(found, ","), entries) +} + +// validAzureAccount mirrors Azure's naming rules: 3-24 lowercase alphanumerics. +func validAzureAccount(s string) bool { + if len(s) < 3 || len(s) > 24 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9') { + return false + } + } + return true } // Target is one provider listing endpoint for a candidate name. diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go index 4fe0b7f..042a385 100644 --- a/internal/discover/discover_test.go +++ b/internal/discover/discover_test.go @@ -17,6 +17,19 @@ func TestCandidatesAreExactLabelsOnly(t *testing.T) { } } +func TestValidAzureAccount(t *testing.T) { + for _, ok := range []string{"myaccount", "abc123", "a1b2c3d4e5f6g7h8i9j0k1l2"} { + if !validAzureAccount(ok) { + t.Fatalf("%q should be a valid Azure account name", ok) + } + } + for _, bad := range []string{"ab", "has-hyphen", "UPPER", "with.dot", "a1b2c3d4e5f6g7h8i9j0k1l2m3"} { + if validAzureAccount(bad) { + t.Fatalf("%q should be rejected as an Azure account name", bad) + } + } +} + func TestBaseLabelsSkipsNoise(t *testing.T) { got := baseLabels("www.example.com") if len(got) != 1 || got[0] != "example" {