discover Azure Blob containers by account name

Azure has two levels: the account maps to DNS (<account>.blob.core.windows.net)
and anonymous access is per container with no anonymous container enumeration.
Discovery now tests incoming labels as Azure account names (DNS existence) and,
for accounts that exist, probes a small set of common containers, indexing any
that list anonymously. CONTENT still never downloaded.

Also document that docker compose down -v wipes all discovered assets; use
stop/start to keep the pgdata volume.

Live: connectsystem.blob.core.windows.net indexed (33 objects).

README.md

@@ -150,7 +150,11 @@ docker compose up --build
 That starts `postgres` and the stateless `ctcloud` container. The application
 container runs as non-root, has a read-only root filesystem, drops all Linux
 capabilities, sets CPU/memory/pids limits and mounts no application volume. The
-only persistent state is PostgreSQL (CT checkpoints + cloud assets).
+only persistent state is PostgreSQL (CT checkpoints + cloud assets + index).
+
+> **Findings accumulate over time.** Use `docker compose stop` / `docker compose
+> start` (or plain `up -d`) to keep the `pgdata` volume. `docker compose down -v`
+> deletes the volume and every discovered asset with it.
 
 Endpoints on port `9090`, bound to `127.0.0.1` only:
 
@@ -324,9 +328,13 @@ engine that:
 3. indexes every bucket that returns 200 (keys/sizes/dates only).
 
 It is aggressively bounded: a global `BUCKET_SCAN_QPS` limit, a 12h per-name
-dedup cache, and a random-sampled domain queue. Azure cannot be enumerated this
+dedup cache, and a random-sampled domain queue.
-way (account names are the namespace and containers need names), so Azure is
+
-handled from CT accounts plus a container wordlist, and R2 is never listable.
+Azure is handled separately because it has two levels: the account name maps to
+DNS (`<account>.blob.core.windows.net`), and anonymous access is per container
+with no anonymous container enumeration — so for accounts that exist we probe a
+small set of common container names. R2 is never listable and is only probed for
+objects.
 
 This is what actually yields open buckets. Of the buckets found, S3 and GCS
 return full object listings; Azure needs a known container; R2 only serves

internal/discover/discover.go

@@ -160,6 +160,92 @@ func (d *Discoverer) testName(ctx context.Context, name string) {
 		}
 		d.test(ctx, t)
 	}
+	d.scanAzure(ctx, name)
+}
+
+// scanAzure handles Azure's two-level model. The account name maps to DNS
+// (<account>.blob.core.windows.net), so a nonexistent account fails the first
+// request and we stop. Anonymous access is per container and Microsoft offers no
+// anonymous container enumeration, so the container names are the one place a
+// wordlist is unavoidable.
+func (d *Discoverer) scanAzure(ctx context.Context, account string) {
+	if !validAzureAccount(account) {
+		return
+	}
+	key := "az:" + account
+	if _, dup := d.seen.Get(key); dup {
+		return
+	}
+	d.seen.Set(key, true, 12*time.Hour)
+
+	host := account + ".blob.core.windows.net"
+	if !d.acquire(ctx) {
+		return
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/?comp=list", nil)
+	if err != nil {
+		return
+	}
+	resp, err := d.client.Do(req)
+	if err != nil {
+		return // account does not exist (NXDOMAIN) or is unreachable
+	}
+	_, _ = io.CopyN(io.Discard, resp.Body, 512)
+	resp.Body.Close()
+
+	// Account exists. Try known/common containers for anonymous listing.
+	var (
+		entries []model.Entry
+		found   []string
+	)
+	for _, container := range index.AzureContainers(account) {
+		if ctx.Err() != nil {
+			return
+		}
+		if d.cfg.Metrics != nil {
+			d.cfg.Metrics.BucketCandidates.Add(1)
+		}
+		if !d.acquire(ctx) {
+			return
+		}
+		c := model.Candidate{
+			Hostname: host, Original: host, Resource: account, Container: container,
+			Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
+			ResourceSpecific: true, Source: model.SourceDiscovery,
+		}
+		e, err := d.cfg.Indexer.List(ctx, c)
+		if err != nil || len(e) == 0 {
+			continue
+		}
+		entries = append(entries, e...)
+		found = append(found, container)
+	}
+	if len(entries) == 0 || d.cfg.OnOpen == nil {
+		return
+	}
+	if d.cfg.Metrics != nil {
+		d.cfg.Metrics.BucketOpen.Add(1)
+	}
+	c := model.Candidate{
+		Hostname: host, Original: host, Resource: account,
+		Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
+		ResourceSpecific: true, Source: model.SourceDiscovery,
+	}
+	d.cfg.OnOpen(ctx, c, "azure public container(s): "+strings.Join(found, ","), entries)
+}
+
+// validAzureAccount mirrors Azure's naming rules: 3-24 lowercase alphanumerics.
+func validAzureAccount(s string) bool {
+	if len(s) < 3 || len(s) > 24 {
+		return false
+	}
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9') {
+			return false
+		}
+	}
+	return true
 }
 
 // Target is one provider listing endpoint for a candidate name.

internal/discover/discover_test.go

@@ -17,6 +17,19 @@ func TestCandidatesAreExactLabelsOnly(t *testing.T) {
 	}
 }
 
+func TestValidAzureAccount(t *testing.T) {
+	for _, ok := range []string{"myaccount", "abc123", "a1b2c3d4e5f6g7h8i9j0k1l2"} {
+		if !validAzureAccount(ok) {
+			t.Fatalf("%q should be a valid Azure account name", ok)
+		}
+	}
+	for _, bad := range []string{"ab", "has-hyphen", "UPPER", "with.dot", "a1b2c3d4e5f6g7h8i9j0k1l2m3"} {
+		if validAzureAccount(bad) {
+			t.Fatalf("%q should be rejected as an Azure account name", bad)
+		}
+	}
+}
+
 func TestBaseLabelsSkipsNoise(t *testing.T) {
 	got := baseLabels("www.example.com")
 	if len(got) != 1 || got[0] != "example" {