provider capability model, on-demand scan/backfill, more providers

Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
  object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
  object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.

Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.

Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.

On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.

Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.

DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.

Verified live against real public resources:
  veteran1.nyc3.digitaloceanspaces.com            listable, entries
  bucket-01.s3-eu-west-1.amazonaws.com            listable, entries
  tartugeo-podcast.storage.googleapis.com         listable, entries
  eroticfeeldev.blob.core.windows.net/images      listable, entries

internal/index/azure_test.go

@@ -0,0 +1,46 @@
+package index
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func TestListAzureContainerDiscovery(t *testing.T) {
+	// Account listing is unavailable; only the "images" container is public.
+	blobXML := `<?xml version="1.0" encoding="utf-8"?><EnumerationResults ContainerName=".../images"><Blobs>
+	<Blob><Name>a.gif</Name><Properties><Content-Length>123</Content-Length><Content-Type>image/gif</Content-Type><Last-Modified>Thu, 06 Nov 2025 13:05:52 GMT</Last-Modified><Etag>0x1</Etag></Properties></Blob>
+	</Blobs></EnumerationResults>`
+	x, _ := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/images" && r.URL.Query().Get("comp") == "list" {
+			_, _ = w.Write([]byte(blobXML))
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	// The test server host is 127.0.0.1:port; listAzure is called with c.Hostname.
+	entries, err := x.List(context.Background(), model.Candidate{
+		Hostname: strings.TrimPrefix(x.gcsBase, "http://"), Resource: "acct",
+		Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 || entries[0].Name != "images/a.gif" || entries[0].Size != 123 {
+		t.Fatalf("unexpected entries: %+v", entries)
+	}
+}
+
+func TestAzureContainersIncludesAccount(t *testing.T) {
+	got := AzureContainers("myappdev")
+	joined := fmt.Sprint(got)
+	for _, want := range []string{"images", "myappdev", "myapp"} {
+		if !strings.Contains(joined, want) {
+			t.Fatalf("AzureContainers missing %q: %v", want, got)
+		}
+	}
+}

internal/index/index.go

@@ -68,9 +68,13 @@ func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, e
 	case model.ProviderAzure:
 		switch c.Service {
 		case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
-			return x.listAzure(ctx, c.Hostname)
+			return x.listAzure(ctx, c.Hostname, c.Resource, c.Container)
 		}
 	case model.ProviderCloudflare:
+		// R2 public URLs serve objects but never allow anonymous listing.
+		if strings.HasSuffix(c.Hostname, ".r2.dev") || strings.HasSuffix(c.Hostname, ".cloudflarestorage.com") {
+			return nil, nil
+		}
 		return x.listS3(ctx, c.Hostname)
 	case model.ProviderDigitalOcean:
 		if c.Service == model.ServiceObjectStorage {
@@ -243,38 +247,73 @@ type azBlobs struct {
 	} `xml:"Blobs"`
 }
 
-func (x *Indexer) listAzure(ctx context.Context, host string) ([]model.Entry, error) {
-	resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list")
-	if err != nil {
-		return nil, err
+// AzureContainers returns candidate container names to probe. Anonymous access
+// in Azure is configured per container and the container name is not part of the
+// hostname, so discovery needs a small dictionary plus account-derived guesses.
+// This is a deliberate, bounded heuristic.
+func AzureContainers(account string) []string {
+	common := []string{
+		"$web", "images", "img", "static", "assets", "media", "public", "www", "web",
+		"files", "uploads", "data", "content", "docs", "downloads", "resources",
+		"backup", "logs", "blog", "site", "app", "store", "photos", "videos",
 	}
-	var page azContainers
-	err = xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page)
-	resp.Body.Close()
-	if err != nil {
-		return nil, err
+	seen := map[string]bool{}
+	out := make([]string, 0, len(common)+4)
+	add := func(s string) {
+		if s != "" && !seen[s] {
+			seen[s] = true
+			out = append(out, s)
+		}
+	}
+	for _, c := range common {
+		add(c)
+	}
+	if account != "" {
+		add(account)
+		for _, suffix := range []string{"dev", "prod", "test", "staging", "storage", "st", "sa"} {
+			if strings.HasSuffix(account, suffix) && len(account) > len(suffix) {
+				add(strings.TrimSuffix(account, suffix))
+			}
+		}
+	}
+	return out
+}
+
+func (x *Indexer) listAzure(ctx context.Context, host, account, hint string) ([]model.Entry, error) {
+	containers := []string{}
+	if hint != "" {
+		containers = []string{hint} // container known from a URL path
+	}
+	if len(containers) == 0 {
+		if resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list"); err == nil {
+			var page azContainers
+			if xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) == nil {
+				for _, c := range page.Containers.Container {
+					containers = append(containers, c.Name)
+				}
+			}
+			resp.Body.Close()
+		}
+	}
+	if len(containers) == 0 {
+		containers = AzureContainers(account)
 	}
 	var entries []model.Entry
-	for _, c := range page.Containers.Container {
-		entries = append(entries, model.Entry{Kind: "container", Name: c.Name})
-		if len(entries) >= x.maxObjects {
-			return entries, nil
-		}
-		// List a bounded number of blobs from each container.
-		blobs, err := x.listAzureContainer(ctx, host, c.Name, x.maxObjects-len(entries))
+	for _, name := range containers {
+		blobs, err := x.listAzureContainer(ctx, host, name, x.maxObjects-len(entries))
 		if err != nil {
 			continue
 		}
 		entries = append(entries, blobs...)
 		if len(entries) >= x.maxObjects {
-			return entries, nil
+			break
 		}
 	}
 	return entries, nil
 }
 
 func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) {
-	resp, err := x.get(ctx, x.scheme+"://"+host+"/"+url.PathEscape(container)+"?restype=container&comp=list")
+	resp, err := x.get(ctx, x.scheme+"://"+host+"/"+container+"?restype=container&comp=list")
 	if err != nil {
 		return nil, err
 	}