StasonJatham
62a7d2eb2e
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
47 lines
1.6 KiB
Go
47 lines
1.6 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
)
|
|
|
|
func TestListAzureContainerDiscovery(t *testing.T) {
|
|
// Account listing is unavailable; only the "images" container is public.
|
|
blobXML := `<?xml version="1.0" encoding="utf-8"?><EnumerationResults ContainerName=".../images"><Blobs>
|
|
<Blob><Name>a.gif</Name><Properties><Content-Length>123</Content-Length><Content-Type>image/gif</Content-Type><Last-Modified>Thu, 06 Nov 2025 13:05:52 GMT</Last-Modified><Etag>0x1</Etag></Properties></Blob>
|
|
</Blobs></EnumerationResults>`
|
|
x, _ := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/images" && r.URL.Query().Get("comp") == "list" {
|
|
_, _ = w.Write([]byte(blobXML))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}))
|
|
// The test server host is 127.0.0.1:port; listAzure is called with c.Hostname.
|
|
entries, err := x.List(context.Background(), model.Candidate{
|
|
Hostname: strings.TrimPrefix(x.gcsBase, "http://"), Resource: "acct",
|
|
Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(entries) != 1 || entries[0].Name != "images/a.gif" || entries[0].Size != 123 {
|
|
t.Fatalf("unexpected entries: %+v", entries)
|
|
}
|
|
}
|
|
|
|
func TestAzureContainersIncludesAccount(t *testing.T) {
|
|
got := AzureContainers("myappdev")
|
|
joined := fmt.Sprint(got)
|
|
for _, want := range []string{"images", "myappdev", "myapp"} {
|
|
if !strings.Contains(joined, want) {
|
|
t.Fatalf("AzureContainers missing %q: %v", want, got)
|
|
}
|
|
}
|
|
}
|