provider capability model, on-demand scan/backfill, more providers

Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
  object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
  object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.

Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.

Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.

On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.

Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.

DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.

Verified live against real public resources:
  veteran1.nyc3.digitaloceanspaces.com            listable, entries
  bucket-01.s3-eu-west-1.amazonaws.com            listable, entries
  tartugeo-podcast.storage.googleapis.com         listable, entries
  eroticfeeldev.blob.core.windows.net/images      listable, entries

internal/probe/probe.go

@@ -3,11 +3,16 @@
 // unauthenticated TCP connect. It never sends credentials, downloads objects,
 // mutates anything, runs database queries, or attempts authentication.
 //
-// "readable" means an anonymous request was answered with data: for object
-// stores we use the vendor's anonymous list endpoint, for Firebase the REST
-// data endpoint, etc. For database endpoints there is no anonymous read concept,
-// so the best honest signal is whether the endpoint is publicly resolvable and
-// its port is open; we report that as reachable, never as readable.
+// Readability is reported as separate capabilities, because they differ by
+// provider:
+//
+//	exists          the hostname resolves / the service answers at all
+//	object_readable an anonymous GET of a known object/key succeeds
+//	listable        the provider's anonymous list API returns the object names
+//
+// Only a listable resource can be enumerated and indexed. Azure grants access
+// per container ("Blob" = object read only, "Container" = listable), and
+// Cloudflare R2 public URLs serve objects but never allow anonymous listing.
 package probe
 
 import (
@@ -20,6 +25,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
@@ -28,8 +34,30 @@ import (
 	"tea.karlcom.de/karl/bucketeer/internal/model"
 )
 
-// UpdateFunc records the observed exposure state and a short factual detail.
-type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState, detail string)
+// Result is the outcome of probing one candidate.
+type Result struct {
+	Exposure       model.ExposureState
+	Detail         string
+	Listable       bool
+	ObjectReadable bool
+}
+
+func res(state model.ExposureState, detail string) Result {
+	return Result{Exposure: state, Detail: detail}
+}
+
+// listRead marks a resource whose names can be enumerated anonymously.
+func listRead(detail string) Result {
+	return Result{Exposure: model.ExposurePublicRead, Detail: detail, Listable: true, ObjectReadable: true}
+}
+
+// objRead marks a resource with a publicly readable object but no listing.
+func objRead(detail string) Result {
+	return Result{Exposure: model.ExposurePublicRead, Detail: detail, ObjectReadable: true}
+}
+
+// UpdateFunc records the observed capabilities and a short factual detail.
+type UpdateFunc func(ctx context.Context, c model.Candidate, r Result)
 
 type Config struct {
 	Workers int
@@ -39,8 +67,8 @@ type Config struct {
 	Metrics *metrics.Metrics
 	Update  UpdateFunc
 
-	// Indexer, when set, lists metadata for resources confirmed public_read and
-	// hands the entries to OnIndex. Contents are never downloaded.
+	// Indexer, when set, lists metadata for listable resources and hands the
+	// entries to OnIndex. Contents are never downloaded.
 	Indexer *index.Indexer
 	OnIndex func(ctx context.Context, c model.Candidate, entries []model.Entry)
 }
@@ -54,10 +82,8 @@ type Prober struct {
 }
 
 // sampleQueue is a memory-bounded queue that keeps a random sample of recent
-// submissions rather than a FIFO. With FIFO, one high-volume hostname template
-// (e.g. *.mrap.accesspoint.s3-global.amazonaws.com) fills the entire queue and
-// no other asset type is ever probed. Random replacement means every template
-// gets sampled over time. It is not on the certificate hot path.
+// submissions rather than a FIFO, so one high-volume hostname template cannot
+// starve every other asset type. It is not on the certificate hot path.
 type sampleQueue struct {
 	mu     sync.Mutex
 	items  []model.Candidate
@@ -147,8 +173,7 @@ func (p *Prober) Start(ctx context.Context) {
 }
 
 // Submit enqueues a candidate without blocking. When the sample is full the
-// oldest-slot candidate is replaced at random; this is load shedding, not an
-// error, so it is not counted as a drop.
+// oldest-slot candidate is replaced at random: load shedding, not an error.
 func (p *Prober) Submit(c model.Candidate) {
 	if p.cfg.Metrics != nil {
 		p.cfg.Metrics.ProbeAttempts.Add(1)
@@ -180,14 +205,15 @@ func (p *Prober) worker(ctx context.Context) {
 			if !ok {
 				break
 			}
-			state, detail := p.Probe(ctx, c)
-			if p.cfg.Metrics != nil && state == model.ExposurePublicRead {
+			r := p.ProbeDetailed(ctx, c)
+			if p.cfg.Metrics != nil && r.Exposure == model.ExposurePublicRead {
 				p.cfg.Metrics.ProbePublic.Add(1)
 			}
 			if p.cfg.Update != nil {
-				p.cfg.Update(ctx, c, state, detail)
+				p.cfg.Update(ctx, c, r)
 			}
-			if state == model.ExposurePublicRead && p.cfg.Indexer != nil && p.cfg.OnIndex != nil {
+			// Only listable resources can be enumerated.
+			if r.Listable && p.cfg.Indexer != nil && p.cfg.OnIndex != nil {
 				if entries, err := p.cfg.Indexer.List(ctx, c); err == nil && len(entries) > 0 {
 					p.cfg.OnIndex(ctx, c, entries)
 					if p.cfg.Metrics != nil {
@@ -199,13 +225,18 @@ func (p *Prober) worker(ctx context.Context) {
 	}
 }
 
-// Probe runs the vendor-appropriate check for a single candidate.
+// Probe is the (exposure, detail) convenience view.
 func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
-	// Deep, read-only protocol checks take precedence for database services and
-	// can distinguish "reachable" from "actually readable without credentials".
+	r := p.ProbeDetailed(ctx, c)
+	return r.Exposure, r.Detail
+}
+
+// ProbeDetailed runs the vendor-appropriate check and reports capabilities.
+func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result {
+	// Deep, read-only protocol checks take precedence for database services.
 	if p.cfg.Deep {
-		if state, detail, ok := p.deepDB(ctx, c); ok {
-			return state, detail
+		if r, ok := p.deepDB(ctx, c); ok {
+			return r
 		}
 	}
 	switch c.Provider {
@@ -216,7 +247,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
 		case model.ServiceRDS, model.ServiceDocumentDB:
 			return p.checkTCP(ctx, c.Hostname, 5432)
 		case model.ServiceNeptune:
-			return p.checkTCP(ctx, c.Hostname, 8182)
+			return p.checkHTTP(ctx, "https://"+c.Hostname+"/status")
 		case model.ServiceElastiCache, model.ServiceMemoryDB:
 			return p.checkTCP(ctx, c.Hostname, 6379)
 		case model.ServiceRedshift:
@@ -228,9 +259,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
 		switch c.Service {
 		case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
 			return p.checkAzureStorage(ctx, c)
-		case model.ServiceWebApp, model.ServiceSearch:
-			return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
-		case model.ServiceCosmosDB:
+		case model.ServiceWebApp, model.ServiceSearch, model.ServiceCosmosDB:
 			return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
 		case model.ServiceDatabase: // Azure SQL
 			return p.checkTCP(ctx, c.Hostname, 1433)
@@ -251,7 +280,12 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
 			return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
 		}
 	case model.ProviderCloudflare:
-		return p.checkS3(ctx, c) // R2 is S3-compatible (and r2.dev serves objects)
+		// R2's authenticated S3 API is not anonymous; public URLs (r2.dev or a
+		// custom domain) serve objects but explicitly do not allow listing.
+		if strings.HasSuffix(c.Hostname, ".r2.dev") {
+			return res(model.ExposureReachable, "r2.dev public object host; anonymous listing is not supported")
+		}
+		return p.checkS3(ctx, c)
 	case model.ProviderDigitalOcean:
 		if c.Service == model.ServiceObjectStorage {
 			return p.checkS3(ctx, c)
@@ -263,52 +297,70 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
 		}
 		return p.checkHTTP(ctx, "https://"+c.Hostname+"/rest/v1/")
 	case model.ProviderOracle:
-		return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
+		return p.checkS3(ctx, c) // S3-compatible
 	}
 	return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
 }
 
-// checkAzureStorage probes the account-level container list. Anonymous read is
-// configured per container in Azure ("Public access level"), never at the
-// account level, so a 401/403 here means only that account listing needs auth —
-// it does NOT mean the containers are unreadable. We therefore report reachable
-// rather than auth_required, and only index if account listing is unexpectedly
-// public (which would let us discover container names).
-func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
-	status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list")
-	if err != nil {
-		return classifyError(err)
+// checkAzureStorage: account-level listing is disabled anonymously (403/404).
+// Anonymous access is per container ("Blob" = object read, "Container" =
+// listable), and the container name is not in the hostname, so probe a bounded
+// set of common names, or the known one when a URL path provided it.
+func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) Result {
+	containers := index.AzureContainers(c.Resource)
+	known := c.Container != ""
+	if known {
+		containers = []string{c.Container}
 	}
-	switch {
-	case status >= 200 && status < 300:
-		return model.ExposurePublicRead, "azure account-level anonymous listing succeeded"
-	case status == http.StatusUnauthorized || status == http.StatusForbidden:
-		return model.ExposureReachable, "azure account listing requires auth; per-container readability needs a container name"
-	default:
-		return classifyStatus(status, "azure")
+	for _, name := range containers {
+		status, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"?restype=container&comp=list")
+		if err != nil {
+			continue
+		}
+		if status >= 200 && status < 300 {
+			return listRead("azure container '" + name + "' is publicly listable")
+		}
+		// "Blob" mode: object readable, container not listable.
+		for _, key := range []string{"index.html", "robots.txt"} {
+			if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"/"+key); err == nil && s >= 200 && s < 300 {
+				return objRead("azure container '" + name + "' objects readable but not listable")
+			}
+		}
 	}
+	if status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list"); err == nil && status >= 200 && status < 300 {
+		return listRead("azure account-level anonymous listing succeeded")
+	}
+	return res(model.ExposureReachable, "azure account listing requires auth; no public container found")
 }
 
-// checkS3 queries the S3 ListObjectsV2 anonymous endpoint. A 200 means the
-// bucket allows anonymous listing (public read); 403 means it exists but is not
-// public; 404 means no such bucket. 301/307 carries the real region.
-func (p *Prober) checkS3(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
+// checkS3 queries ListObjectsV2 anonymously. 200 = listable; 403 with a readable
+// known object = object readable only; 404 = no such bucket; 301/307 = region.
+func (p *Prober) checkS3(ctx context.Context, c model.Candidate) Result {
 	status, header, err := p.do(ctx, "https://"+c.Hostname+"/?list-type=2&max-keys=1")
 	if err != nil {
-		return classifyError(err)
+		return wrap(classifyError(err))
 	}
 	if status == http.StatusMovedPermanently || status == http.StatusTemporaryRedirect || status == http.StatusPermanentRedirect {
 		if region := header.Get("x-amz-bucket-region"); region != "" {
-			return model.ExposureReachable, "s3 redirect region=" + region
+			return res(model.ExposureReachable, "s3 redirect region="+region)
 		}
-		return model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status)
+		return res(model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status))
 	}
-	return classifyStatus(status, "s3")
+	if status >= 200 && status < 300 {
+		return listRead(fmt.Sprintf("s3 status=%d anonymous listing allowed", status))
+	}
+	if status == http.StatusForbidden || status == http.StatusUnauthorized {
+		for _, key := range []string{"index.html", "robots.txt"} {
+			if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+key); err == nil && s >= 200 && s < 300 {
+				return objRead("s3 object readable (" + key + ") although listing is denied")
+			}
+		}
+	}
+	return wrap(classifyStatus(status, "s3"))
 }
 
-// checkGCS uses the public JSON API to test object listing, which requires the
-// bucket to grant allUsers read access.
-func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
+// checkGCS uses the public JSON API object list (requires storage.objects.list).
+func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) Result {
 	bucket := c.Resource
 	if bucket == "" {
 		bucket = c.Hostname
@@ -316,40 +368,52 @@ func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.Exposur
 	api := "https://storage.googleapis.com/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1"
 	status, _, err := p.do(ctx, api)
 	if err != nil {
-		return classifyError(err)
+		return wrap(classifyError(err))
 	}
-	return classifyStatus(status, "gcs")
+	if status >= 200 && status < 300 {
+		return listRead(fmt.Sprintf("gcs status=%d anonymous listing allowed", status))
+	}
+	if status == http.StatusForbidden || status == http.StatusUnauthorized {
+		for _, key := range []string{"index.html", "robots.txt"} {
+			obj := "https://storage.googleapis.com/" + url.PathEscape(bucket) + "/" + key
+			if s, _, err := p.do(ctx, obj); err == nil && s >= 200 && s < 300 {
+				return objRead("gcs object readable (" + key + ") although listing is denied")
+			}
+		}
+	}
+	return wrap(classifyStatus(status, "gcs"))
 }
 
-// checkFirebase queries the Realtime Database REST endpoint with shallow=true so
-// only keys are returned. 200 means the security rules allow anonymous reads.
-func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
+// checkFirebase queries the Realtime Database REST endpoint with shallow=true.
+func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) Result {
 	status, _, err := p.do(ctx, "https://"+c.Hostname+"/.json?shallow=true")
 	if err != nil {
-		return classifyError(err)
+		return wrap(classifyError(err))
 	}
-	return classifyStatus(status, "firebase")
+	if status >= 200 && status < 300 {
+		return listRead("firebase rules allow anonymous read")
+	}
+	return wrap(classifyStatus(status, "firebase"))
 }
 
-func (p *Prober) checkHTTP(ctx context.Context, endpoint string) (model.ExposureState, string) {
+func (p *Prober) checkHTTP(ctx context.Context, endpoint string) Result {
 	status, _, err := p.do(ctx, endpoint)
 	if err != nil {
-		return classifyError(err)
+		return wrap(classifyError(err))
 	}
-	return classifyStatus(status, "")
+	return wrap(classifyStatus(status, ""))
 }
 
 // checkTCP reports whether a database endpoint is publicly resolvable and its
-// port is open. It intentionally does not attempt any protocol handshake or
-// authentication, so it can never be reported as readable.
-func (p *Prober) checkTCP(ctx context.Context, host string, port int) (model.ExposureState, string) {
+// port is open. It never authenticates, so it can never be readable.
+func (p *Prober) checkTCP(ctx context.Context, host string, port int) Result {
 	d := net.Dialer{Timeout: p.cfg.Timeout}
 	conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, fmt.Sprint(port)))
 	if err != nil {
-		return classifyError(err)
+		return wrap(classifyError(err))
 	}
 	_ = conn.Close()
-	return model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port)
+	return res(model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port))
 }
 
 // do issues a single GET and reads at most a tiny prefix of the body before
@@ -368,6 +432,10 @@ func (p *Prober) do(ctx context.Context, endpoint string) (int, http.Header, err
 	return resp.StatusCode, resp.Header, nil
 }
 
+func wrap(state model.ExposureState, detail string) Result {
+	return Result{Exposure: state, Detail: detail}
+}
+
 func classifyStatus(status int, tag string) (model.ExposureState, string) {
 	prefix := ""
 	if tag != "" {