StasonJatham
62a7d2eb2e
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
107 lines
3.4 KiB
Go
107 lines
3.4 KiB
Go
package scan
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/dns"
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
"tea.karlcom.de/karl/bucketeer/internal/probe"
|
|
)
|
|
|
|
type fakeProber struct {
|
|
last model.Candidate
|
|
state model.ExposureState
|
|
detail string
|
|
listable bool
|
|
obj bool
|
|
}
|
|
|
|
func (f *fakeProber) ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result {
|
|
f.last = c
|
|
return probe.Result{Exposure: f.state, Detail: f.detail, Listable: f.listable, ObjectReadable: f.obj}
|
|
}
|
|
|
|
type fakeLister struct{ entries []model.Entry }
|
|
|
|
func (f *fakeLister) List(ctx context.Context, c model.Candidate) ([]model.Entry, error) {
|
|
return f.entries, nil
|
|
}
|
|
|
|
type fakeEnricher struct{ chain []string }
|
|
|
|
func (f *fakeEnricher) Resolve(ctx context.Context, host string) (dns.Result, bool) {
|
|
return dns.Result{Chain: append([]string{host}, f.chain...)}, true
|
|
}
|
|
|
|
type fakeStore struct {
|
|
ensured []model.Candidate
|
|
entries int
|
|
}
|
|
|
|
func (f *fakeStore) EnsureAsset(ctx context.Context, c model.Candidate) (int64, error) {
|
|
f.ensured = append(f.ensured, c)
|
|
return int64(len(f.ensured)), nil
|
|
}
|
|
|
|
func (f *fakeStore) ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error) {
|
|
f.entries = len(entries)
|
|
return len(entries), nil
|
|
}
|
|
|
|
func (f *fakeStore) ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStore) UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error {
|
|
return nil
|
|
}
|
|
|
|
func TestScanDirectReadable(t *testing.T) {
|
|
pb := &fakeProber{state: model.ExposurePublicRead, detail: "s3 status=200", listable: true}
|
|
st := &fakeStore{}
|
|
s := &Scanner{Prober: pb, Indexer: &fakeLister{entries: []model.Entry{{Name: "a.txt"}, {Name: "b.txt"}}}, Store: st}
|
|
res := s.Scan(context.Background(), "bucket-01.s3-eu-west-1.amazonaws.com")
|
|
if len(res) != 1 || res[0].Exposure != "public_read" || res[0].Entries != 2 || len(res[0].Sample) != 2 {
|
|
t.Fatalf("unexpected result: %+v", res)
|
|
}
|
|
}
|
|
|
|
func TestScanAzureContainerPath(t *testing.T) {
|
|
pb := &fakeProber{state: model.ExposurePublicRead, detail: "azure container 'images' is publicly listable", listable: true}
|
|
s := &Scanner{Prober: pb, Indexer: &fakeLister{}, Store: &fakeStore{}}
|
|
res := s.Scan(context.Background(), "https://eroticfeeldev.blob.core.windows.net/images")
|
|
if len(res) != 1 || res[0].Service != "blob_storage" {
|
|
t.Fatalf("unexpected result: %+v", res)
|
|
}
|
|
if pb.last.Container != "images" {
|
|
t.Fatalf("container hint not passed: %+v", pb.last)
|
|
}
|
|
}
|
|
|
|
func TestScanViaCNAME(t *testing.T) {
|
|
pb := &fakeProber{state: model.ExposurePublicRead, listable: true}
|
|
st := &fakeStore{}
|
|
s := &Scanner{
|
|
Prober: pb,
|
|
Indexer: &fakeLister{},
|
|
Store: st,
|
|
Enricher: &fakeEnricher{chain: []string{"veteran1.nyc3.digitaloceanspaces.com"}},
|
|
}
|
|
res := s.Scan(context.Background(), "cdn.mydomain.example")
|
|
if len(res) != 1 || res[0].Provider != "digitalocean" {
|
|
t.Fatalf("expected digitalocean via CNAME, got %+v", res)
|
|
}
|
|
if len(res[0].CNAMEChain) < 2 {
|
|
t.Fatalf("expected CNAME chain, got %+v", res[0].CNAMEChain)
|
|
}
|
|
}
|
|
|
|
func TestScanUnknown(t *testing.T) {
|
|
s := &Scanner{Prober: &fakeProber{}, Store: &fakeStore{}}
|
|
res := s.Scan(context.Background(), "www.example.com")
|
|
if len(res) != 1 || res[0].Error == "" {
|
|
t.Fatalf("expected error result, got %+v", res)
|
|
}
|
|
}
|