provider capability model, on-demand scan/backfill, more providers

Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
  object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
  object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.

Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.

Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.

On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.

Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.

DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.

Verified live against real public resources:
  veteran1.nyc3.digitaloceanspaces.com            listable, entries
  bucket-01.s3-eu-west-1.amazonaws.com            listable, entries
  tartugeo-podcast.storage.googleapis.com         listable, entries
  eroticfeeldev.blob.core.windows.net/images      listable, entries
This commit is contained in:
StasonJatham committed 2026-09-22 14:47:49 +02:00
1 parent a366905a43
commit 62a7d2eb2e
23 files changed
+936 -165

No files matched your search

+58 -19
View File
@@ -68,9 +68,13 @@ func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, e
case model.ProviderAzure:
switch c.Service {
case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
return x.listAzure(ctx, c.Hostname)
return x.listAzure(ctx, c.Hostname, c.Resource, c.Container)
}
case model.ProviderCloudflare:
// R2 public URLs serve objects but never allow anonymous listing.
if strings.HasSuffix(c.Hostname, ".r2.dev") || strings.HasSuffix(c.Hostname, ".cloudflarestorage.com") {
return nil, nil
}
return x.listS3(ctx, c.Hostname)
case model.ProviderDigitalOcean:
if c.Service == model.ServiceObjectStorage {
@@ -243,38 +247,73 @@ type azBlobs struct {
} `xml:"Blobs"`
}
func (x *Indexer) listAzure(ctx context.Context, host string) ([]model.Entry, error) {
resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list")
if err != nil {
return nil, err
// AzureContainers returns candidate container names to probe. Anonymous access
// in Azure is configured per container and the container name is not part of the
// hostname, so discovery needs a small dictionary plus account-derived guesses.
// This is a deliberate, bounded heuristic.
func AzureContainers(account string) []string {
common := []string{
"$web", "images", "img", "static", "assets", "media", "public", "www", "web",
"files", "uploads", "data", "content", "docs", "downloads", "resources",
"backup", "logs", "blog", "site", "app", "store", "photos", "videos",
}
var page azContainers
err = xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page)
resp.Body.Close()
if err != nil {
return nil, err
seen := map[string]bool{}
out := make([]string, 0, len(common)+4)
add := func(s string) {
if s != "" && !seen[s] {
seen[s] = true
out = append(out, s)
}
}
for _, c := range common {
add(c)
}
if account != "" {
add(account)
for _, suffix := range []string{"dev", "prod", "test", "staging", "storage", "st", "sa"} {
if strings.HasSuffix(account, suffix) && len(account) > len(suffix) {
add(strings.TrimSuffix(account, suffix))
}
}
}
return out
}
func (x *Indexer) listAzure(ctx context.Context, host, account, hint string) ([]model.Entry, error) {
containers := []string{}
if hint != "" {
containers = []string{hint} // container known from a URL path
}
if len(containers) == 0 {
if resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list"); err == nil {
var page azContainers
if xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) == nil {
for _, c := range page.Containers.Container {
containers = append(containers, c.Name)
}
}
resp.Body.Close()
}
}
if len(containers) == 0 {
containers = AzureContainers(account)
}
var entries []model.Entry
for _, c := range page.Containers.Container {
entries = append(entries, model.Entry{Kind: "container", Name: c.Name})
if len(entries) >= x.maxObjects {
return entries, nil
}
// List a bounded number of blobs from each container.
blobs, err := x.listAzureContainer(ctx, host, c.Name, x.maxObjects-len(entries))
for _, name := range containers {
blobs, err := x.listAzureContainer(ctx, host, name, x.maxObjects-len(entries))
if err != nil {
continue
}
entries = append(entries, blobs...)
if len(entries) >= x.maxObjects {
return entries, nil
break
}
}
return entries, nil
}
func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) {
resp, err := x.get(ctx, x.scheme+"://"+host+"/"+url.PathEscape(container)+"?restype=container&comp=list")
resp, err := x.get(ctx, x.scheme+"://"+host+"/"+container+"?restype=container&comp=list")
if err != nil {
return nil, err
}