provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
This commit is contained in:
1 parent
a366905a43
commit
62a7d2eb2e
23 files changed
+936
-165
No files matched your search
+58
-19
@@ -68,9 +68,13 @@ func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, e
|
||||
case model.ProviderAzure:
|
||||
switch c.Service {
|
||||
case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
|
||||
return x.listAzure(ctx, c.Hostname)
|
||||
return x.listAzure(ctx, c.Hostname, c.Resource, c.Container)
|
||||
}
|
||||
case model.ProviderCloudflare:
|
||||
// R2 public URLs serve objects but never allow anonymous listing.
|
||||
if strings.HasSuffix(c.Hostname, ".r2.dev") || strings.HasSuffix(c.Hostname, ".cloudflarestorage.com") {
|
||||
return nil, nil
|
||||
}
|
||||
return x.listS3(ctx, c.Hostname)
|
||||
case model.ProviderDigitalOcean:
|
||||
if c.Service == model.ServiceObjectStorage {
|
||||
@@ -243,38 +247,73 @@ type azBlobs struct {
|
||||
} `xml:"Blobs"`
|
||||
}
|
||||
|
||||
func (x *Indexer) listAzure(ctx context.Context, host string) ([]model.Entry, error) {
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// AzureContainers returns candidate container names to probe. Anonymous access
|
||||
// in Azure is configured per container and the container name is not part of the
|
||||
// hostname, so discovery needs a small dictionary plus account-derived guesses.
|
||||
// This is a deliberate, bounded heuristic.
|
||||
func AzureContainers(account string) []string {
|
||||
common := []string{
|
||||
"$web", "images", "img", "static", "assets", "media", "public", "www", "web",
|
||||
"files", "uploads", "data", "content", "docs", "downloads", "resources",
|
||||
"backup", "logs", "blog", "site", "app", "store", "photos", "videos",
|
||||
}
|
||||
var page azContainers
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(common)+4)
|
||||
add := func(s string) {
|
||||
if s != "" && !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
for _, c := range common {
|
||||
add(c)
|
||||
}
|
||||
if account != "" {
|
||||
add(account)
|
||||
for _, suffix := range []string{"dev", "prod", "test", "staging", "storage", "st", "sa"} {
|
||||
if strings.HasSuffix(account, suffix) && len(account) > len(suffix) {
|
||||
add(strings.TrimSuffix(account, suffix))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (x *Indexer) listAzure(ctx context.Context, host, account, hint string) ([]model.Entry, error) {
|
||||
containers := []string{}
|
||||
if hint != "" {
|
||||
containers = []string{hint} // container known from a URL path
|
||||
}
|
||||
if len(containers) == 0 {
|
||||
if resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list"); err == nil {
|
||||
var page azContainers
|
||||
if xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) == nil {
|
||||
for _, c := range page.Containers.Container {
|
||||
containers = append(containers, c.Name)
|
||||
}
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
if len(containers) == 0 {
|
||||
containers = AzureContainers(account)
|
||||
}
|
||||
var entries []model.Entry
|
||||
for _, c := range page.Containers.Container {
|
||||
entries = append(entries, model.Entry{Kind: "container", Name: c.Name})
|
||||
if len(entries) >= x.maxObjects {
|
||||
return entries, nil
|
||||
}
|
||||
// List a bounded number of blobs from each container.
|
||||
blobs, err := x.listAzureContainer(ctx, host, c.Name, x.maxObjects-len(entries))
|
||||
for _, name := range containers {
|
||||
blobs, err := x.listAzureContainer(ctx, host, name, x.maxObjects-len(entries))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, blobs...)
|
||||
if len(entries) >= x.maxObjects {
|
||||
return entries, nil
|
||||
break
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) {
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/"+url.PathEscape(container)+"?restype=container&comp=list")
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/"+container+"?restype=container&comp=list")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user