diff --git a/README.md b/README.md index efdac84..a0d5af0 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,11 @@ docker compose up --build That starts `postgres` and the stateless `ctcloud` container. The application container runs as non-root, has a read-only root filesystem, drops all Linux capabilities, sets CPU/memory/pids limits and mounts no application volume. The -only persistent state is PostgreSQL (CT checkpoints + cloud assets). +only persistent state is PostgreSQL (CT checkpoints + cloud assets + index). + +> **Findings accumulate over time.** Use `docker compose stop` / `docker compose +> start` (or plain `up -d`) to keep the `pgdata` volume. `docker compose down -v` +> deletes the volume and every discovered asset with it. Endpoints on port `9090`, bound to `127.0.0.1` only: @@ -324,9 +328,13 @@ engine that: 3. indexes every bucket that returns 200 (keys/sizes/dates only). It is aggressively bounded: a global `BUCKET_SCAN_QPS` limit, a 12h per-name -dedup cache, and a random-sampled domain queue. Azure cannot be enumerated this -way (account names are the namespace and containers need names), so Azure is -handled from CT accounts plus a container wordlist, and R2 is never listable. +dedup cache, and a random-sampled domain queue. + +Azure is handled separately because it has two levels: the account name maps to +DNS (`.blob.core.windows.net`), and anonymous access is per container +with no anonymous container enumeration — so for accounts that exist we probe a +small set of common container names. R2 is never listable and is only probed for +objects. This is what actually yields open buckets. Of the buckets found, S3 and GCS return full object listings; Azure needs a known container; R2 only serves diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 173ada2..19563fc 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -160,6 +160,92 @@ func (d *Discoverer) testName(ctx context.Context, name string) { } d.test(ctx, t) } + d.scanAzure(ctx, name) +} + +// scanAzure handles Azure's two-level model. The account name maps to DNS +// (.blob.core.windows.net), so a nonexistent account fails the first +// request and we stop. Anonymous access is per container and Microsoft offers no +// anonymous container enumeration, so the container names are the one place a +// wordlist is unavoidable. +func (d *Discoverer) scanAzure(ctx context.Context, account string) { + if !validAzureAccount(account) { + return + } + key := "az:" + account + if _, dup := d.seen.Get(key); dup { + return + } + d.seen.Set(key, true, 12*time.Hour) + + host := account + ".blob.core.windows.net" + if !d.acquire(ctx) { + return + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/?comp=list", nil) + if err != nil { + return + } + resp, err := d.client.Do(req) + if err != nil { + return // account does not exist (NXDOMAIN) or is unreachable + } + _, _ = io.CopyN(io.Discard, resp.Body, 512) + resp.Body.Close() + + // Account exists. Try known/common containers for anonymous listing. + var ( + entries []model.Entry + found []string + ) + for _, container := range index.AzureContainers(account) { + if ctx.Err() != nil { + return + } + if d.cfg.Metrics != nil { + d.cfg.Metrics.BucketCandidates.Add(1) + } + if !d.acquire(ctx) { + return + } + c := model.Candidate{ + Hostname: host, Original: host, Resource: account, Container: container, + Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, + ResourceSpecific: true, Source: model.SourceDiscovery, + } + e, err := d.cfg.Indexer.List(ctx, c) + if err != nil || len(e) == 0 { + continue + } + entries = append(entries, e...) + found = append(found, container) + } + if len(entries) == 0 || d.cfg.OnOpen == nil { + return + } + if d.cfg.Metrics != nil { + d.cfg.Metrics.BucketOpen.Add(1) + } + c := model.Candidate{ + Hostname: host, Original: host, Resource: account, + Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, + ResourceSpecific: true, Source: model.SourceDiscovery, + } + d.cfg.OnOpen(ctx, c, "azure public container(s): "+strings.Join(found, ","), entries) +} + +// validAzureAccount mirrors Azure's naming rules: 3-24 lowercase alphanumerics. +func validAzureAccount(s string) bool { + if len(s) < 3 || len(s) > 24 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9') { + return false + } + } + return true } // Target is one provider listing endpoint for a candidate name. diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go index 4fe0b7f..042a385 100644 --- a/internal/discover/discover_test.go +++ b/internal/discover/discover_test.go @@ -17,6 +17,19 @@ func TestCandidatesAreExactLabelsOnly(t *testing.T) { } } +func TestValidAzureAccount(t *testing.T) { + for _, ok := range []string{"myaccount", "abc123", "a1b2c3d4e5f6g7h8i9j0k1l2"} { + if !validAzureAccount(ok) { + t.Fatalf("%q should be a valid Azure account name", ok) + } + } + for _, bad := range []string{"ab", "has-hyphen", "UPPER", "with.dot", "a1b2c3d4e5f6g7h8i9j0k1l2m3"} { + if validAzureAccount(bad) { + t.Fatalf("%q should be rejected as an Azure account name", bad) + } + } +} + func TestBaseLabelsSkipsNoise(t *testing.T) { got := baseLabels("www.example.com") if len(got) != 1 || got[0] != "example" {