diff --git a/README.md b/README.md index 8caa1f3..14b014c 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ isolated so it can be rebased. ## Supported provider patterns **AWS** — S3 (`.s3[.dualstack]..amazonaws.com`, -`s3-website[-]`), S3 access points (`s3-accesspoint[-fips]`, +`.s3-.amazonaws.com` legacy form, `s3-website[-]`), +S3 access points (`s3-accesspoint[-fips]`, `dualstack`), multi-region access points (`mrap.accesspoint.s3-global`), S3 Express (`s3express-*`), RDS/Aurora/proxy (`*.rds.amazonaws.com`), DocumentDB, Redshift (provisioned + serverless), OpenSearch (`search-*` public / `vpc-*` diff --git a/cmd/ctcloud/main.go b/cmd/ctcloud/main.go index 4df3f46..9f405ac 100644 --- a/cmd/ctcloud/main.go +++ b/cmd/ctcloud/main.go @@ -26,6 +26,7 @@ import ( "tea.karlcom.de/karl/bucketeer/internal/model" "tea.karlcom.de/karl/bucketeer/internal/pipeline" "tea.karlcom.de/karl/bucketeer/internal/probe" + "tea.karlcom.de/karl/bucketeer/internal/scan" "tea.karlcom.de/karl/bucketeer/internal/store" "tea.karlcom.de/karl/bucketeer/internal/webui" ) @@ -128,6 +129,14 @@ func run() error { Metrics: m, } + // One indexer is shared by the CT prober and the on-demand scanner. + indexer := index.New(cfg.IndexMaxObjects, 2*cfg.ProbeTimeout) + + // Scanner prober: a separate, never-started Prober used synchronously by the + // on-demand scan endpoint (Probe is stateless). + scanProber := probe.New(probe.Config{Workers: 1, Timeout: cfg.ProbeTimeout, Deep: cfg.ProbeDeep, Metrics: m}) + scanner := &scan.Scanner{Prober: scanProber, Indexer: indexer, Enricher: enricher, Store: st} + var prober *probe.Prober if cfg.ActiveProbing { pcfg := probe.Config{ @@ -136,12 +145,13 @@ func run() error { Queue: cfg.ProbeQueueSize, Deep: cfg.ProbeDeep, Metrics: m, - Update: func(uctx context.Context, c model.Candidate, state model.ExposureState, detail string) { - _ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state, detail) + Update: func(uctx context.Context, c model.Candidate, r probe.Result) { + _ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, + r.Exposure, r.Detail, r.Listable, r.ObjectReadable) }, } if cfg.IndexEnabled { - pcfg.Indexer = index.New(cfg.IndexMaxObjects, 2*cfg.ProbeTimeout) + pcfg.Indexer = indexer pcfg.OnIndex = func(uctx context.Context, c model.Candidate, entries []model.Entry) { if n, err := st.ReplaceEntries(uctx, c.Provider.String(), c.Service.String(), c.Hostname, entries); err != nil { slog.Warn("index write failed", "event", "index_error", "host", c.Hostname, "error", err) @@ -197,7 +207,7 @@ func run() error { monitorDone <- monitor.Run(ctx, monitorCfg) }() - ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass}) + ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass, Scanner: scanner}) httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady) go logStats(ctx, m, p, st, resolver, prober, cfg.StatsInterval) diff --git a/internal/classify/aws.go b/internal/classify/aws.go index daf98b3..cdaf6f5 100644 --- a/internal/classify/aws.go +++ b/internal/classify/aws.go @@ -144,5 +144,17 @@ func classifyAWSS3(s *scanner) (model.Match, bool) { } return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true } + // Legacy regional form: .s3-.amazonaws.com (e.g. s3-eu-west-1) + if n == 4 { + svc := s.label(n - 3) + if region, ok := strings.CutPrefix(svc, "s3-"); ok && strings.Contains(region, "-") && + !strings.HasPrefix(svc, "s3-website") && !strings.HasPrefix(svc, "s3-accesspoint") && + !strings.HasPrefix(svc, "s3express-") { + res := s.label(0) + if res != "s3" { + return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, region), true + } + } + } return model.Match{}, false } diff --git a/internal/classify/classify_test.go b/internal/classify/classify_test.go index 534d878..74ed021 100644 --- a/internal/classify/classify_test.go +++ b/internal/classify/classify_test.go @@ -61,6 +61,7 @@ func TestClassifyMatches(t *testing.T) { {"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}}, {"x.mrap.accesspoint.s3-global.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "", true, 0}}, {"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}}, + {"bucket-01.s3-eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket-01", "eu-west-1", true, 0}}, // AWS databases {"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}}, {"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}}, diff --git a/internal/config/config.go b/internal/config/config.go index 6afb11d..9fa7f98 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -60,7 +60,7 @@ func Load() (*Config, error) { DatabaseURL: os.Getenv("DATABASE_URL"), DBMaxConns: envInt("DB_MAX_CONNS", 8), DNSServers: envList("DNS_SERVERS", nil), - DNSWorkers: envInt("DNS_WORKERS", 16), + DNSWorkers: envInt("DNS_WORKERS", 64), DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18), DedupSize: envInt("DEDUP_SIZE", 1<<18), @@ -68,7 +68,7 @@ func Load() (*Config, error) { DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), - DNSQueueSize: envInt("DNS_QUEUE_SIZE", 65536), + DNSQueueSize: envInt("DNS_QUEUE_SIZE", 262144), DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), DBBatchSize: envInt("DB_BATCH_SIZE", 500), DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9145be6..6a78a41 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,7 +14,7 @@ func TestLoadDefaults(t *testing.T) { if c.DatabaseURL != "postgres://localhost/ctcloud" { t.Fatalf("database url = %q", c.DatabaseURL) } - if c.DNSWorkers != 16 || c.DNSMaxCNAMEdepth != 8 { + if c.DNSWorkers != 64 || c.DNSMaxCNAMEdepth != 8 { t.Fatalf("unexpected DNS defaults: %+v", c) } if c.ActiveProbing { diff --git a/internal/dns/resolver.go b/internal/dns/resolver.go index dfff3de..a70d393 100644 --- a/internal/dns/resolver.go +++ b/internal/dns/resolver.go @@ -88,6 +88,75 @@ type Enricher struct { Metrics *metrics.Metrics } +// ResolveCNAME follows just the CNAME chain (no A/AAAA), for the high-volume +// enrichment path where only the targets are classified. Cached separately from +// full results. +func (e *Enricher) ResolveCNAME(ctx context.Context, host string) ([]string, bool) { + if e.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, e.Timeout) + defer cancel() + } + key := "c:" + host + if e.Cache != nil { + if res, ok := e.Cache.Get(key); ok { + if e.Metrics != nil { + e.Metrics.DNSCacheHits.Add(1) + } + return res.Chain, true + } + } + if e.Metrics != nil { + e.Metrics.DNSQueries.Add(1) + } + chain, err := Follow(ctx, e.Resolver, host, e.MaxDepth) + if err != nil && len(chain) == 1 { + if e.Metrics != nil && !definitive(err) { + e.Metrics.DNSErrors.Add(1) + } + if e.Cache != nil { + e.Cache.Set(key, Result{}, e.NegativeTTL) + } + return nil, false + } + if e.Cache != nil { + e.Cache.Set(key, Result{Chain: chain}, e.PositiveTTL) + } + return chain, true +} + +// ResolveIPs resolves A/AAAA for a host, cached. +func (e *Enricher) ResolveIPs(ctx context.Context, host string) ([]net.IP, bool) { + if e.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, e.Timeout) + defer cancel() + } + key := "ip:" + host + if e.Cache != nil { + if res, ok := e.Cache.Get(key); ok { + if e.Metrics != nil { + e.Metrics.DNSCacheHits.Add(1) + } + return res.IPs, true + } + } + ips, err := e.Resolver.LookupIP(ctx, host) + if err != nil || len(ips) == 0 { + if e.Metrics != nil { + e.Metrics.DNSErrors.Add(1) + } + if e.Cache != nil { + e.Cache.Set(key, Result{}, e.NegativeTTL) + } + return nil, false + } + if e.Cache != nil { + e.Cache.Set(key, Result{IPs: ips}, e.PositiveTTL) + } + return ips, true +} + // Resolve returns the result for host. ok is false when the lookup failed; // failures (including "no such host") are negatively cached for NegativeTTL. func (e *Enricher) Resolve(ctx context.Context, host string) (Result, bool) { diff --git a/internal/index/azure_test.go b/internal/index/azure_test.go new file mode 100644 index 0000000..7c35acd --- /dev/null +++ b/internal/index/azure_test.go @@ -0,0 +1,46 @@ +package index + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +func TestListAzureContainerDiscovery(t *testing.T) { + // Account listing is unavailable; only the "images" container is public. + blobXML := ` + a.gif123image/gifThu, 06 Nov 2025 13:05:52 GMT0x1 + ` + x, _ := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/images" && r.URL.Query().Get("comp") == "list" { + _, _ = w.Write([]byte(blobXML)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + // The test server host is 127.0.0.1:port; listAzure is called with c.Hostname. + entries, err := x.List(context.Background(), model.Candidate{ + Hostname: strings.TrimPrefix(x.gcsBase, "http://"), Resource: "acct", + Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, + }) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name != "images/a.gif" || entries[0].Size != 123 { + t.Fatalf("unexpected entries: %+v", entries) + } +} + +func TestAzureContainersIncludesAccount(t *testing.T) { + got := AzureContainers("myappdev") + joined := fmt.Sprint(got) + for _, want := range []string{"images", "myappdev", "myapp"} { + if !strings.Contains(joined, want) { + t.Fatalf("AzureContainers missing %q: %v", want, got) + } + } +} diff --git a/internal/index/index.go b/internal/index/index.go index 2ab1838..932e9ff 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -68,9 +68,13 @@ func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, e case model.ProviderAzure: switch c.Service { case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage: - return x.listAzure(ctx, c.Hostname) + return x.listAzure(ctx, c.Hostname, c.Resource, c.Container) } case model.ProviderCloudflare: + // R2 public URLs serve objects but never allow anonymous listing. + if strings.HasSuffix(c.Hostname, ".r2.dev") || strings.HasSuffix(c.Hostname, ".cloudflarestorage.com") { + return nil, nil + } return x.listS3(ctx, c.Hostname) case model.ProviderDigitalOcean: if c.Service == model.ServiceObjectStorage { @@ -243,38 +247,73 @@ type azBlobs struct { } `xml:"Blobs"` } -func (x *Indexer) listAzure(ctx context.Context, host string) ([]model.Entry, error) { - resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list") - if err != nil { - return nil, err +// AzureContainers returns candidate container names to probe. Anonymous access +// in Azure is configured per container and the container name is not part of the +// hostname, so discovery needs a small dictionary plus account-derived guesses. +// This is a deliberate, bounded heuristic. +func AzureContainers(account string) []string { + common := []string{ + "$web", "images", "img", "static", "assets", "media", "public", "www", "web", + "files", "uploads", "data", "content", "docs", "downloads", "resources", + "backup", "logs", "blog", "site", "app", "store", "photos", "videos", } - var page azContainers - err = xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) - resp.Body.Close() - if err != nil { - return nil, err + seen := map[string]bool{} + out := make([]string, 0, len(common)+4) + add := func(s string) { + if s != "" && !seen[s] { + seen[s] = true + out = append(out, s) + } + } + for _, c := range common { + add(c) + } + if account != "" { + add(account) + for _, suffix := range []string{"dev", "prod", "test", "staging", "storage", "st", "sa"} { + if strings.HasSuffix(account, suffix) && len(account) > len(suffix) { + add(strings.TrimSuffix(account, suffix)) + } + } + } + return out +} + +func (x *Indexer) listAzure(ctx context.Context, host, account, hint string) ([]model.Entry, error) { + containers := []string{} + if hint != "" { + containers = []string{hint} // container known from a URL path + } + if len(containers) == 0 { + if resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list"); err == nil { + var page azContainers + if xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) == nil { + for _, c := range page.Containers.Container { + containers = append(containers, c.Name) + } + } + resp.Body.Close() + } + } + if len(containers) == 0 { + containers = AzureContainers(account) } var entries []model.Entry - for _, c := range page.Containers.Container { - entries = append(entries, model.Entry{Kind: "container", Name: c.Name}) - if len(entries) >= x.maxObjects { - return entries, nil - } - // List a bounded number of blobs from each container. - blobs, err := x.listAzureContainer(ctx, host, c.Name, x.maxObjects-len(entries)) + for _, name := range containers { + blobs, err := x.listAzureContainer(ctx, host, name, x.maxObjects-len(entries)) if err != nil { continue } entries = append(entries, blobs...) if len(entries) >= x.maxObjects { - return entries, nil + break } } return entries, nil } func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) { - resp, err := x.get(ctx, x.scheme+"://"+host+"/"+url.PathEscape(container)+"?restype=container&comp=list") + resp, err := x.get(ctx, x.scheme+"://"+host+"/"+container+"?restype=container&comp=list") if err != nil { return nil, err } diff --git a/internal/model/model.go b/internal/model/model.go index 3b95868..5ca272a 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -125,6 +125,7 @@ const ( SourceDirectCT SourceCNAME SourceDNSAlias + SourceManual ) func (s Source) String() string { @@ -135,6 +136,8 @@ func (s Source) String() string { return "cname" case SourceDNSAlias: return "dns_alias" + case SourceManual: + return "manual" default: return "unknown" } @@ -196,6 +199,38 @@ func (e ExposureState) String() string { } } +// ParseProvider maps a stored provider string back to its enum. +func ParseProvider(s string) Provider { + switch s { + case "aws": + return ProviderAWS + case "azure": + return ProviderAzure + case "gcp": + return ProviderGCP + case "cloudflare": + return ProviderCloudflare + case "digitalocean": + return ProviderDigitalOcean + case "supabase": + return ProviderSupabase + case "oracle": + return ProviderOracle + default: + return ProviderUnknown + } +} + +// ParseService maps a stored service string back to its enum. +func ParseService(s string) Service { + for svc := ServiceUnknown + 1; svc <= ServiceWebApp; svc++ { + if svc.String() == s { + return svc + } + } + return ServiceUnknown +} + // Match is the compact result of classifying a single hostname. The string // fields are substrings of the input hostname (zero-copy). type Match struct { @@ -207,6 +242,15 @@ type Match struct { Resource string } +// ProbeTarget is a persisted asset selected for re-probing/backfill. +type ProbeTarget struct { + Provider string + Service string + Hostname string + Resource string + Region string +} + // Entry is one indexed item of a readable resource (object key, container, // prefix, database index, root key). It is metadata only — never the content. type Entry struct { @@ -224,6 +268,7 @@ type Candidate struct { Original string // original CT hostname (equal to Hostname for direct matches) Region string Resource string + Container string // optional storage container/share hint (Azure) Provider Provider Service Service Source Source diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index e65bae6..6fcae68 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -255,11 +255,13 @@ func (p *Pipeline) dnsWorker(ctx context.Context) { } func (p *Pipeline) enrichHost(ctx context.Context, host string) { - res, ok := p.cfg.Enricher.Resolve(ctx, host) + // CNAME-only during enrichment: this is the high-volume path and A/AAAA is + // not needed to classify the target. IPs are resolved only for matches. + chain, ok := p.cfg.Enricher.ResolveCNAME(ctx, host) if !ok { return } - for _, target := range res.Chain[1:] { + for _, target := range chain[1:] { m, matched := classify.Classify(target) if !matched || !m.ResourceSpecific { continue @@ -273,9 +275,11 @@ func (p *Pipeline) enrichHost(ctx context.Context, host string) { } f := Finding{ Candidate: candidateFrom(target, host, m, model.SourceCNAME), - CNAMEChain: res.Chain, + CNAMEChain: chain, + } + if ips, ok := p.cfg.Enricher.ResolveIPs(ctx, target); ok { + appendIPs(&f, ips) } - appendIPs(&f, res.IPs) p.pushDB(ctx, f) } } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index a019921..acfbd1f 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -149,7 +149,7 @@ func TestCNAMEMatch(t *testing.T) { col := &collector{} r := &fakeResolver{ cnames: map[string]string{"files.example.com": "files.s3.amazonaws.com"}, - ips: map[string][]net.IP{"files.example.com": {net.ParseIP("1.2.3.4")}}, + ips: map[string][]net.IP{"files.s3.amazonaws.com": {net.ParseIP("1.2.3.4")}}, } m := metrics.New() p := New(Config{Enricher: newEnricher(r), Write: col.write, Metrics: m, BatchSize: 10}) @@ -180,7 +180,7 @@ func TestCNAMEMatchIPv6(t *testing.T) { col := &collector{} r := &fakeResolver{ cnames: map[string]string{"db.example.com": "mydb.cluster-abc.us-east-1.rds.amazonaws.com"}, - ips: map[string][]net.IP{"db.example.com": {net.ParseIP("2606:4700::1111")}}, + ips: map[string][]net.IP{"mydb.cluster-abc.us-east-1.rds.amazonaws.com": {net.ParseIP("2606:4700::1111")}}, } p := New(Config{Enricher: newEnricher(r), Write: col.write, BatchSize: 10}) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/probe/probe.go b/internal/probe/probe.go index 035c087..0766d37 100644 --- a/internal/probe/probe.go +++ b/internal/probe/probe.go @@ -3,11 +3,16 @@ // unauthenticated TCP connect. It never sends credentials, downloads objects, // mutates anything, runs database queries, or attempts authentication. // -// "readable" means an anonymous request was answered with data: for object -// stores we use the vendor's anonymous list endpoint, for Firebase the REST -// data endpoint, etc. For database endpoints there is no anonymous read concept, -// so the best honest signal is whether the endpoint is publicly resolvable and -// its port is open; we report that as reachable, never as readable. +// Readability is reported as separate capabilities, because they differ by +// provider: +// +// exists the hostname resolves / the service answers at all +// object_readable an anonymous GET of a known object/key succeeds +// listable the provider's anonymous list API returns the object names +// +// Only a listable resource can be enumerated and indexed. Azure grants access +// per container ("Blob" = object read only, "Container" = listable), and +// Cloudflare R2 public URLs serve objects but never allow anonymous listing. package probe import ( @@ -20,6 +25,7 @@ import ( "net" "net/http" "net/url" + "strings" "sync" "time" @@ -28,8 +34,30 @@ import ( "tea.karlcom.de/karl/bucketeer/internal/model" ) -// UpdateFunc records the observed exposure state and a short factual detail. -type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState, detail string) +// Result is the outcome of probing one candidate. +type Result struct { + Exposure model.ExposureState + Detail string + Listable bool + ObjectReadable bool +} + +func res(state model.ExposureState, detail string) Result { + return Result{Exposure: state, Detail: detail} +} + +// listRead marks a resource whose names can be enumerated anonymously. +func listRead(detail string) Result { + return Result{Exposure: model.ExposurePublicRead, Detail: detail, Listable: true, ObjectReadable: true} +} + +// objRead marks a resource with a publicly readable object but no listing. +func objRead(detail string) Result { + return Result{Exposure: model.ExposurePublicRead, Detail: detail, ObjectReadable: true} +} + +// UpdateFunc records the observed capabilities and a short factual detail. +type UpdateFunc func(ctx context.Context, c model.Candidate, r Result) type Config struct { Workers int @@ -39,8 +67,8 @@ type Config struct { Metrics *metrics.Metrics Update UpdateFunc - // Indexer, when set, lists metadata for resources confirmed public_read and - // hands the entries to OnIndex. Contents are never downloaded. + // Indexer, when set, lists metadata for listable resources and hands the + // entries to OnIndex. Contents are never downloaded. Indexer *index.Indexer OnIndex func(ctx context.Context, c model.Candidate, entries []model.Entry) } @@ -54,10 +82,8 @@ type Prober struct { } // sampleQueue is a memory-bounded queue that keeps a random sample of recent -// submissions rather than a FIFO. With FIFO, one high-volume hostname template -// (e.g. *.mrap.accesspoint.s3-global.amazonaws.com) fills the entire queue and -// no other asset type is ever probed. Random replacement means every template -// gets sampled over time. It is not on the certificate hot path. +// submissions rather than a FIFO, so one high-volume hostname template cannot +// starve every other asset type. It is not on the certificate hot path. type sampleQueue struct { mu sync.Mutex items []model.Candidate @@ -147,8 +173,7 @@ func (p *Prober) Start(ctx context.Context) { } // Submit enqueues a candidate without blocking. When the sample is full the -// oldest-slot candidate is replaced at random; this is load shedding, not an -// error, so it is not counted as a drop. +// oldest-slot candidate is replaced at random: load shedding, not an error. func (p *Prober) Submit(c model.Candidate) { if p.cfg.Metrics != nil { p.cfg.Metrics.ProbeAttempts.Add(1) @@ -180,14 +205,15 @@ func (p *Prober) worker(ctx context.Context) { if !ok { break } - state, detail := p.Probe(ctx, c) - if p.cfg.Metrics != nil && state == model.ExposurePublicRead { + r := p.ProbeDetailed(ctx, c) + if p.cfg.Metrics != nil && r.Exposure == model.ExposurePublicRead { p.cfg.Metrics.ProbePublic.Add(1) } if p.cfg.Update != nil { - p.cfg.Update(ctx, c, state, detail) + p.cfg.Update(ctx, c, r) } - if state == model.ExposurePublicRead && p.cfg.Indexer != nil && p.cfg.OnIndex != nil { + // Only listable resources can be enumerated. + if r.Listable && p.cfg.Indexer != nil && p.cfg.OnIndex != nil { if entries, err := p.cfg.Indexer.List(ctx, c); err == nil && len(entries) > 0 { p.cfg.OnIndex(ctx, c, entries) if p.cfg.Metrics != nil { @@ -199,13 +225,18 @@ func (p *Prober) worker(ctx context.Context) { } } -// Probe runs the vendor-appropriate check for a single candidate. +// Probe is the (exposure, detail) convenience view. func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) { - // Deep, read-only protocol checks take precedence for database services and - // can distinguish "reachable" from "actually readable without credentials". + r := p.ProbeDetailed(ctx, c) + return r.Exposure, r.Detail +} + +// ProbeDetailed runs the vendor-appropriate check and reports capabilities. +func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result { + // Deep, read-only protocol checks take precedence for database services. if p.cfg.Deep { - if state, detail, ok := p.deepDB(ctx, c); ok { - return state, detail + if r, ok := p.deepDB(ctx, c); ok { + return r } } switch c.Provider { @@ -216,7 +247,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt case model.ServiceRDS, model.ServiceDocumentDB: return p.checkTCP(ctx, c.Hostname, 5432) case model.ServiceNeptune: - return p.checkTCP(ctx, c.Hostname, 8182) + return p.checkHTTP(ctx, "https://"+c.Hostname+"/status") case model.ServiceElastiCache, model.ServiceMemoryDB: return p.checkTCP(ctx, c.Hostname, 6379) case model.ServiceRedshift: @@ -228,9 +259,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt switch c.Service { case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage: return p.checkAzureStorage(ctx, c) - case model.ServiceWebApp, model.ServiceSearch: - return p.checkHTTP(ctx, "https://"+c.Hostname+"/") - case model.ServiceCosmosDB: + case model.ServiceWebApp, model.ServiceSearch, model.ServiceCosmosDB: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") case model.ServiceDatabase: // Azure SQL return p.checkTCP(ctx, c.Hostname, 1433) @@ -251,7 +280,12 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } case model.ProviderCloudflare: - return p.checkS3(ctx, c) // R2 is S3-compatible (and r2.dev serves objects) + // R2's authenticated S3 API is not anonymous; public URLs (r2.dev or a + // custom domain) serve objects but explicitly do not allow listing. + if strings.HasSuffix(c.Hostname, ".r2.dev") { + return res(model.ExposureReachable, "r2.dev public object host; anonymous listing is not supported") + } + return p.checkS3(ctx, c) case model.ProviderDigitalOcean: if c.Service == model.ServiceObjectStorage { return p.checkS3(ctx, c) @@ -263,52 +297,70 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt } return p.checkHTTP(ctx, "https://"+c.Hostname+"/rest/v1/") case model.ProviderOracle: - return p.checkHTTP(ctx, "https://"+c.Hostname+"/") + return p.checkS3(ctx, c) // S3-compatible } return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } -// checkAzureStorage probes the account-level container list. Anonymous read is -// configured per container in Azure ("Public access level"), never at the -// account level, so a 401/403 here means only that account listing needs auth — -// it does NOT mean the containers are unreadable. We therefore report reachable -// rather than auth_required, and only index if account listing is unexpectedly -// public (which would let us discover container names). -func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) (model.ExposureState, string) { - status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list") - if err != nil { - return classifyError(err) +// checkAzureStorage: account-level listing is disabled anonymously (403/404). +// Anonymous access is per container ("Blob" = object read, "Container" = +// listable), and the container name is not in the hostname, so probe a bounded +// set of common names, or the known one when a URL path provided it. +func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) Result { + containers := index.AzureContainers(c.Resource) + known := c.Container != "" + if known { + containers = []string{c.Container} } - switch { - case status >= 200 && status < 300: - return model.ExposurePublicRead, "azure account-level anonymous listing succeeded" - case status == http.StatusUnauthorized || status == http.StatusForbidden: - return model.ExposureReachable, "azure account listing requires auth; per-container readability needs a container name" - default: - return classifyStatus(status, "azure") + for _, name := range containers { + status, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"?restype=container&comp=list") + if err != nil { + continue + } + if status >= 200 && status < 300 { + return listRead("azure container '" + name + "' is publicly listable") + } + // "Blob" mode: object readable, container not listable. + for _, key := range []string{"index.html", "robots.txt"} { + if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"/"+key); err == nil && s >= 200 && s < 300 { + return objRead("azure container '" + name + "' objects readable but not listable") + } + } } + if status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list"); err == nil && status >= 200 && status < 300 { + return listRead("azure account-level anonymous listing succeeded") + } + return res(model.ExposureReachable, "azure account listing requires auth; no public container found") } -// checkS3 queries the S3 ListObjectsV2 anonymous endpoint. A 200 means the -// bucket allows anonymous listing (public read); 403 means it exists but is not -// public; 404 means no such bucket. 301/307 carries the real region. -func (p *Prober) checkS3(ctx context.Context, c model.Candidate) (model.ExposureState, string) { +// checkS3 queries ListObjectsV2 anonymously. 200 = listable; 403 with a readable +// known object = object readable only; 404 = no such bucket; 301/307 = region. +func (p *Prober) checkS3(ctx context.Context, c model.Candidate) Result { status, header, err := p.do(ctx, "https://"+c.Hostname+"/?list-type=2&max-keys=1") if err != nil { - return classifyError(err) + return wrap(classifyError(err)) } if status == http.StatusMovedPermanently || status == http.StatusTemporaryRedirect || status == http.StatusPermanentRedirect { if region := header.Get("x-amz-bucket-region"); region != "" { - return model.ExposureReachable, "s3 redirect region=" + region + return res(model.ExposureReachable, "s3 redirect region="+region) } - return model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status) + return res(model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status)) } - return classifyStatus(status, "s3") + if status >= 200 && status < 300 { + return listRead(fmt.Sprintf("s3 status=%d anonymous listing allowed", status)) + } + if status == http.StatusForbidden || status == http.StatusUnauthorized { + for _, key := range []string{"index.html", "robots.txt"} { + if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+key); err == nil && s >= 200 && s < 300 { + return objRead("s3 object readable (" + key + ") although listing is denied") + } + } + } + return wrap(classifyStatus(status, "s3")) } -// checkGCS uses the public JSON API to test object listing, which requires the -// bucket to grant allUsers read access. -func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.ExposureState, string) { +// checkGCS uses the public JSON API object list (requires storage.objects.list). +func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) Result { bucket := c.Resource if bucket == "" { bucket = c.Hostname @@ -316,40 +368,52 @@ func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.Exposur api := "https://storage.googleapis.com/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1" status, _, err := p.do(ctx, api) if err != nil { - return classifyError(err) + return wrap(classifyError(err)) } - return classifyStatus(status, "gcs") + if status >= 200 && status < 300 { + return listRead(fmt.Sprintf("gcs status=%d anonymous listing allowed", status)) + } + if status == http.StatusForbidden || status == http.StatusUnauthorized { + for _, key := range []string{"index.html", "robots.txt"} { + obj := "https://storage.googleapis.com/" + url.PathEscape(bucket) + "/" + key + if s, _, err := p.do(ctx, obj); err == nil && s >= 200 && s < 300 { + return objRead("gcs object readable (" + key + ") although listing is denied") + } + } + } + return wrap(classifyStatus(status, "gcs")) } -// checkFirebase queries the Realtime Database REST endpoint with shallow=true so -// only keys are returned. 200 means the security rules allow anonymous reads. -func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) (model.ExposureState, string) { +// checkFirebase queries the Realtime Database REST endpoint with shallow=true. +func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) Result { status, _, err := p.do(ctx, "https://"+c.Hostname+"/.json?shallow=true") if err != nil { - return classifyError(err) + return wrap(classifyError(err)) } - return classifyStatus(status, "firebase") + if status >= 200 && status < 300 { + return listRead("firebase rules allow anonymous read") + } + return wrap(classifyStatus(status, "firebase")) } -func (p *Prober) checkHTTP(ctx context.Context, endpoint string) (model.ExposureState, string) { +func (p *Prober) checkHTTP(ctx context.Context, endpoint string) Result { status, _, err := p.do(ctx, endpoint) if err != nil { - return classifyError(err) + return wrap(classifyError(err)) } - return classifyStatus(status, "") + return wrap(classifyStatus(status, "")) } // checkTCP reports whether a database endpoint is publicly resolvable and its -// port is open. It intentionally does not attempt any protocol handshake or -// authentication, so it can never be reported as readable. -func (p *Prober) checkTCP(ctx context.Context, host string, port int) (model.ExposureState, string) { +// port is open. It never authenticates, so it can never be readable. +func (p *Prober) checkTCP(ctx context.Context, host string, port int) Result { d := net.Dialer{Timeout: p.cfg.Timeout} conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, fmt.Sprint(port))) if err != nil { - return classifyError(err) + return wrap(classifyError(err)) } _ = conn.Close() - return model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port) + return res(model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port)) } // do issues a single GET and reads at most a tiny prefix of the body before @@ -368,6 +432,10 @@ func (p *Prober) do(ctx context.Context, endpoint string) (int, http.Header, err return resp.StatusCode, resp.Header, nil } +func wrap(state model.ExposureState, detail string) Result { + return Result{Exposure: state, Detail: detail} +} + func classifyStatus(status int, tag string) (model.ExposureState, string) { prefix := "" if tag != "" { diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go index a70ac29..0d01627 100644 --- a/internal/probe/probe_test.go +++ b/internal/probe/probe_test.go @@ -80,6 +80,24 @@ func TestProbeFirebase(t *testing.T) { } } +func TestProbeAzureContainerDiscovery(t *testing.T) { + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/images" && r.URL.Query().Get("comp") == "list" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + }) + p := New(Config{Workers: 1, Timeout: time.Second}) + host := serverHost(t, h) + state, detail := p.Probe(context.Background(), model.Candidate{ + Hostname: host, Resource: "acct", Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, + }) + if state != model.ExposurePublicRead || !strings.Contains(detail, "images") { + t.Fatalf("got %s (%s), want public_read via container 'images'", state, detail) + } +} + func TestProbeDatabaseTCP(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -97,17 +115,16 @@ func TestProbeDatabaseTCP(t *testing.T) { }() host, port, _ := net.SplitHostPort(ln.Addr().String()) p := New(Config{Workers: 1, Timeout: time.Second}) - state, detail := p.checkTCP(context.Background(), host, mustAtoi(port)) - if state != model.ExposureReachable { - t.Fatalf("got %s (%s), want reachable", state, detail) + r := p.checkTCP(context.Background(), host, mustAtoi(port)) + if r.Exposure != model.ExposureReachable { + t.Fatalf("got %+v, want reachable", r) } - if !strings.Contains(detail, "no authentication attempted") { - t.Fatalf("detail should be explicit about no auth: %q", detail) + if r.ObjectReadable || r.Listable { + t.Fatalf("tcp check must not claim readability: %+v", r) } // A closed port yields unknown, never readable. - state, _ = p.checkTCP(context.Background(), "127.0.0.1", 1) - if state == model.ExposurePublicRead { + if r := p.checkTCP(context.Background(), "127.0.0.1", 1); r.Exposure == model.ExposurePublicRead { t.Fatal("closed port must never be public_read") } } diff --git a/internal/probe/protocols.go b/internal/probe/protocols.go index 637b24b..964e7e8 100644 --- a/internal/probe/protocols.go +++ b/internal/probe/protocols.go @@ -18,7 +18,7 @@ import ( // database endpoint actually answers a read without credentials. No password, // token or credential of any kind is ever sent, and no state is mutated. -func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureState, string, bool) { +func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (Result, bool) { switch c.Service { case model.ServiceRedis, model.ServiceElastiCache, model.ServiceMemoryDB: return p.probeRedis(ctx, c.Hostname, dbPort(c.Service)) @@ -31,7 +31,7 @@ func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureS case model.ServiceNeptune: return p.probeHTTPReadable(ctx, "https://"+c.Hostname+"/status") } - return model.ExposureUnknown, "", false + return Result{}, false } func dbPort(s model.Service) int { @@ -75,7 +75,7 @@ func (p *Prober) dialTLS(ctx context.Context, host string, port int) (net.Conn, } // probeRedis sends PING. Redis answers +PONG when no authentication is required. -func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.ExposureState, string, bool) { +func (p *Prober) probeRedis(ctx context.Context, host string, port int) (Result, bool) { var lastErr error for _, tlsOn := range []bool{false, true} { var ( @@ -99,9 +99,9 @@ func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.E resp := string(buf[:n]) switch { case strings.HasPrefix(resp, "+PONG"): - return model.ExposurePublicRead, "redis PING answered without auth", true + return listRead("redis PING answered without auth"), true case strings.Contains(resp, "NOAUTH"), strings.HasPrefix(resp, "-ERR"), strings.Contains(resp, "DENIED"), strings.Contains(resp, "protected mode"): - return model.ExposureAuthRequired, "redis requires authentication", true + return wrap(model.ExposureAuthRequired, "redis requires authentication"), true } // no/invalid response: try the other transport } @@ -109,17 +109,16 @@ func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.E lastErr = fmt.Errorf("redis: no response") } st, detail := classifyError(lastErr) - return st, detail, true + return wrap(st, detail), true } // probePostgres negotiates TLS if offered, then sends a StartupMessage with no // password. The server's AuthenticationRequest tells us whether it trusts the // connection (AuthenticationOk) or requires a password. -func (p *Prober) probePostgres(ctx context.Context, host string, port int) (model.ExposureState, string, bool) { +func (p *Prober) probePostgres(ctx context.Context, host string, port int) (Result, bool) { conn, err := p.dial(ctx, host, port) if err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } defer conn.Close() _ = conn.SetDeadline(time.Now().Add(p.cfg.Timeout)) @@ -148,14 +147,12 @@ func (p *Prober) probePostgres(ctx context.Context, host string, port int) (mode binary.BigEndian.PutUint32(msg, uint32(4+len(payload))) copy(msg[4:], payload) if _, err := conn.Write(msg); err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } head := make([]byte, 5) if _, err := io.ReadFull(conn, head); err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } kind := head[0] length := int(binary.BigEndian.Uint32(head[1:5])) @@ -165,20 +162,20 @@ func (p *Prober) probePostgres(ctx context.Context, host string, port int) (mode if kind == 'R' && len(body) >= 4 { authType := binary.BigEndian.Uint32(body[:4]) if authType == 0 { - return model.ExposurePublicRead, "postgres AuthenticationOk (trust, no password required)", true + return listRead("postgres AuthenticationOk (trust, no password required)"), true } - return model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType), true + return wrap(model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType)), true } if kind == 'E' { - return model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)", true + return wrap(model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)"), true } } - return model.ExposureReachable, "postgres port open; auth state unclear", true + return wrap(model.ExposureReachable, "postgres port open; auth state unclear"), true } // probeMongo performs the OP_MSG handshake then a read-only listDatabases. A // successful, non-empty reply means the deployment has no authentication. -func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.ExposureState, string, bool) { +func (p *Prober) probeMongo(ctx context.Context, host string, port int) (Result, bool) { var ( conn net.Conn err error @@ -188,8 +185,7 @@ func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.E if err != nil { conn, err = p.dial(ctx, host, port) if err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } } defer conn.Close() @@ -197,33 +193,29 @@ func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.E doc := bsonDoc{"listDatabases": int32(1), "$db": "admin"} if err := writeOPMsg(conn, doc); err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } raw, err := readOPMsg(conn) if err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } ok, errmsg := parseBSONFields(raw) switch { case ok == 1 && errmsg == "": - return model.ExposurePublicRead, "mongodb listDatabases succeeded without auth", true + return listRead("mongodb listDatabases succeeded without auth"), true case strings.Contains(strings.ToLower(errmsg), "auth") || strings.Contains(strings.ToLower(errmsg), "unauthorized"): - return model.ExposureAuthRequired, "mongodb requires authentication", true + return wrap(model.ExposureAuthRequired, "mongodb requires authentication"), true default: - return model.ExposureReachable, "mongodb replied but auth state unclear", true + return wrap(model.ExposureReachable, "mongodb replied but auth state unclear"), true } } -func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (model.ExposureState, string, bool) { +func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (Result, bool) { status, _, err := p.do(ctx, endpoint) if err != nil { - st, d := classifyError(err) - return st, d, true + return wrap(classifyError(err)), true } - st, d := classifyStatus(status, "") - return st, d, true + return wrap(classifyStatus(status, "")), true } // --- minimal BSON/OP_MSG helpers (read-only) --- diff --git a/internal/probe/protocols_test.go b/internal/probe/protocols_test.go index 965c218..7c7b01f 100644 --- a/internal/probe/protocols_test.go +++ b/internal/probe/protocols_test.go @@ -46,9 +46,9 @@ func TestProbeRedisReadable(t *testing.T) { _, _ = c.Write([]byte("+PONG\r\n")) }) p := New(Config{Workers: 1, Timeout: time.Second}) - state, detail, ok := p.probeRedis(context.Background(), host, port) - if !ok || state != model.ExposurePublicRead { - t.Fatalf("got %s (%s) ok=%v, want public_read", state, detail, ok) + r, ok := p.probeRedis(context.Background(), host, port) + if !ok || r.Exposure != model.ExposurePublicRead || !r.ObjectReadable { + t.Fatalf("got %+v ok=%v, want public_read/object_readable", r, ok) } } @@ -59,9 +59,9 @@ func TestProbeRedisAuthRequired(t *testing.T) { _, _ = c.Write([]byte("-NOAUTH Authentication required.\r\n")) }) p := New(Config{Workers: 1, Timeout: time.Second}) - state, _, _ := p.probeRedis(context.Background(), host, port) - if state != model.ExposureAuthRequired { - t.Fatalf("got %s, want auth_required", state) + r, _ := p.probeRedis(context.Background(), host, port) + if r.Exposure != model.ExposureAuthRequired { + t.Fatalf("got %+v, want auth_required", r) } } @@ -82,9 +82,9 @@ func TestProbePostgresTrust(t *testing.T) { _, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 0}) }) p := New(Config{Workers: 1, Timeout: time.Second}) - state, detail, _ := p.probePostgres(context.Background(), host, port) - if state != model.ExposurePublicRead { - t.Fatalf("got %s (%s), want public_read", state, detail) + r, _ := p.probePostgres(context.Background(), host, port) + if r.Exposure != model.ExposurePublicRead || !r.ObjectReadable { + t.Fatalf("got %+v, want public_read/object_readable", r) } } @@ -105,9 +105,9 @@ func TestProbePostgresPassword(t *testing.T) { _, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 5}) }) p := New(Config{Workers: 1, Timeout: time.Second}) - state, _, _ := p.probePostgres(context.Background(), host, port) - if state != model.ExposureAuthRequired { - t.Fatalf("got %s, want auth_required", state) + r, _ := p.probePostgres(context.Background(), host, port) + if r.Exposure != model.ExposureAuthRequired { + t.Fatalf("got %+v, want auth_required", r) } } diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go new file mode 100644 index 0000000..4de6cd4 --- /dev/null +++ b/internal/scan/scanner.go @@ -0,0 +1,214 @@ +// Package scan probes and indexes a user-supplied hostname or URL on demand. +// This is how a specific public bucket that is not necessarily in the CT stream +// gets verified and enumerated. It reuses the same classifier, prober and +// indexer as the CT pipeline, so results are identical. +package scan + +import ( + "context" + "net" + "strings" + + "tea.karlcom.de/karl/bucketeer/internal/classify" + "tea.karlcom.de/karl/bucketeer/internal/dns" + "tea.karlcom.de/karl/bucketeer/internal/model" + "tea.karlcom.de/karl/bucketeer/internal/probe" +) + +// Store is the write subset the scanner needs. +type Store interface { + EnsureAsset(ctx context.Context, c model.Candidate) (int64, error) + ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error) + ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error) + UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error +} + +// The dependencies are interfaces so the scanner is testable. +type Prober interface { + ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result +} + +type Lister interface { + List(ctx context.Context, c model.Candidate) ([]model.Entry, error) +} + +type Enricher interface { + Resolve(ctx context.Context, host string) (dns.Result, bool) +} + +type Scanner struct { + Prober Prober + Indexer Lister + Enricher Enricher + Store Store +} + +// Result is the outcome of scanning one resolved provider resource. +type Result struct { + Target string `json:"target"` + Host string `json:"host"` + Provider string `json:"provider"` + Service string `json:"service"` + Exposure string `json:"exposure"` + Detail string `json:"detail"` + Listable bool `json:"listable"` + ObjectReadable bool `json:"object_readable"` + Entries int `json:"entries"` + Sample []string `json:"sample,omitempty"` + CNAMEChain []string `json:"cname_chain,omitempty"` + Error string `json:"error,omitempty"` +} + +// Scan resolves the target (directly or through its CNAME chain), probes each +// provider resource it maps to, and indexes readable ones. +func (s *Scanner) Scan(ctx context.Context, target string) []Result { + host, path := splitTarget(target) + if host == "" { + return []Result{{Target: target, Error: "empty target"}} + } + + if m, ok := classify.Classify(host); ok { + c := candidateFor(host, host, m) + if path != "" && c.Provider == model.ProviderAzure { + c.Container = firstSegment(path) + } + return []Result{s.run(ctx, c, nil)} + } + + // Follow the CNAME chain and classify every target (custom domains pointing + // at a bucket/CDN are the common case). + if s.Enricher != nil { + res, ok := s.Enricher.Resolve(ctx, host) + if ok { + var out []Result + for _, t := range res.Chain[1:] { + if m, matched := classify.Classify(t); matched && m.ResourceSpecific { + out = append(out, s.run(ctx, candidateFor(t, host, m), res.Chain)) + } + } + if len(out) == 0 { + return []Result{{Target: target, Host: host, CNAMEChain: res.Chain, + Error: "resolved, but no CNAME target matched a known provider pattern"}} + } + return out + } + } + return []Result{{Target: target, Host: host, Error: "host does not match a known provider pattern and has no provider CNAME"}} +} + +// BackfillReport summarizes a re-probe of already-discovered assets. +type BackfillReport struct { + Probed int `json:"probed"` + Readable int `json:"readable"` + Listable int `json:"listable"` + Indexed int `json:"indexed"` + Errors int `json:"errors"` +} + +// Backfill re-probes assets already in the database (optionally filtered by +// exposure_state) with the current checks and indexes any that are readable. +// This recovers findings discovered before a check was improved. +func (s *Scanner) Backfill(ctx context.Context, exposure string, limit int) (BackfillReport, error) { + targets, err := s.Store.ListProbeTargets(ctx, exposure, limit) + if err != nil { + return BackfillReport{}, err + } + var rep BackfillReport + for _, t := range targets { + if err := ctx.Err(); err != nil { + return rep, err + } + c := model.Candidate{ + Hostname: t.Hostname, Original: t.Hostname, Resource: t.Resource, Region: t.Region, + Provider: model.ParseProvider(t.Provider), Service: model.ParseService(t.Service), + Source: model.SourceManual, + } + pr := s.Prober.ProbeDetailed(ctx, c) + rep.Probed++ + if pr.Exposure == model.ExposurePublicRead { + rep.Readable++ + } + if pr.Listable { + rep.Listable++ + } + if err := s.Store.UpdateExposure(ctx, t.Provider, t.Service, t.Hostname, + pr.Exposure, pr.Detail, pr.Listable, pr.ObjectReadable); err != nil { + rep.Errors++ + continue + } + if pr.Listable && s.Indexer != nil { + entries, err := s.Indexer.List(ctx, c) + if err == nil && len(entries) > 0 { + if n, werr := s.Store.ReplaceEntries(ctx, t.Provider, t.Service, t.Hostname, entries); werr == nil { + rep.Indexed += n + } else { + rep.Errors++ + } + } + } + } + return rep, nil +} + +func (s *Scanner) run(ctx context.Context, c model.Candidate, chain []string) Result { + r := Result{Target: c.Original, Host: c.Hostname, Provider: c.Provider.String(), + Service: c.Service.String(), CNAMEChain: chain} + c.Source = model.SourceManual + if _, err := s.Store.EnsureAsset(ctx, c); err != nil { + r.Error = "store: " + err.Error() + return r + } + pr := s.Prober.ProbeDetailed(ctx, c) + r.Exposure, r.Detail = pr.Exposure.String(), pr.Detail + r.Listable, r.ObjectReadable = pr.Listable, pr.ObjectReadable + if pr.Listable && s.Indexer != nil { + entries, err := s.Indexer.List(ctx, c) + if err == nil && len(entries) > 0 { + n, werr := s.Store.ReplaceEntries(ctx, c.Provider.String(), c.Service.String(), c.Hostname, entries) + if werr != nil { + r.Error = "index store: " + werr.Error() + } + r.Entries = n + for i, e := range entries { + if i >= 10 { + break + } + r.Sample = append(r.Sample, e.Name) + } + } + } + return r +} + +func candidateFor(host, original string, m model.Match) model.Candidate { + return model.Candidate{ + Hostname: host, Original: original, Region: m.Region, Resource: m.Resource, + Provider: m.Provider, Service: m.Service, PublicSemantic: m.PublicSemantic, + ResourceSpecific: m.ResourceSpecific, Source: model.SourceManual, + } +} + +// splitTarget accepts "host", "host/path" or a full URL and returns the host and +// optional path (without a leading slash). +func splitTarget(target string) (string, string) { + target = strings.TrimSpace(target) + if i := strings.Index(target, "://"); i >= 0 { + target = target[i+3:] + } + host, path := target, "" + if i := strings.IndexByte(target, '/'); i >= 0 { + host, path = target[:i], target[i+1:] + } + host = strings.TrimSuffix(strings.ToLower(host), ".") + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + return host, path +} + +func firstSegment(path string) string { + if i := strings.IndexByte(path, '/'); i >= 0 { + return path[:i] + } + return path +} diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go new file mode 100644 index 0000000..8615e16 --- /dev/null +++ b/internal/scan/scanner_test.go @@ -0,0 +1,106 @@ +package scan + +import ( + "context" + "testing" + + "tea.karlcom.de/karl/bucketeer/internal/dns" + "tea.karlcom.de/karl/bucketeer/internal/model" + "tea.karlcom.de/karl/bucketeer/internal/probe" +) + +type fakeProber struct { + last model.Candidate + state model.ExposureState + detail string + listable bool + obj bool +} + +func (f *fakeProber) ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result { + f.last = c + return probe.Result{Exposure: f.state, Detail: f.detail, Listable: f.listable, ObjectReadable: f.obj} +} + +type fakeLister struct{ entries []model.Entry } + +func (f *fakeLister) List(ctx context.Context, c model.Candidate) ([]model.Entry, error) { + return f.entries, nil +} + +type fakeEnricher struct{ chain []string } + +func (f *fakeEnricher) Resolve(ctx context.Context, host string) (dns.Result, bool) { + return dns.Result{Chain: append([]string{host}, f.chain...)}, true +} + +type fakeStore struct { + ensured []model.Candidate + entries int +} + +func (f *fakeStore) EnsureAsset(ctx context.Context, c model.Candidate) (int64, error) { + f.ensured = append(f.ensured, c) + return int64(len(f.ensured)), nil +} + +func (f *fakeStore) ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error) { + f.entries = len(entries) + return len(entries), nil +} + +func (f *fakeStore) ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error) { + return nil, nil +} + +func (f *fakeStore) UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error { + return nil +} + +func TestScanDirectReadable(t *testing.T) { + pb := &fakeProber{state: model.ExposurePublicRead, detail: "s3 status=200", listable: true} + st := &fakeStore{} + s := &Scanner{Prober: pb, Indexer: &fakeLister{entries: []model.Entry{{Name: "a.txt"}, {Name: "b.txt"}}}, Store: st} + res := s.Scan(context.Background(), "bucket-01.s3-eu-west-1.amazonaws.com") + if len(res) != 1 || res[0].Exposure != "public_read" || res[0].Entries != 2 || len(res[0].Sample) != 2 { + t.Fatalf("unexpected result: %+v", res) + } +} + +func TestScanAzureContainerPath(t *testing.T) { + pb := &fakeProber{state: model.ExposurePublicRead, detail: "azure container 'images' is publicly listable", listable: true} + s := &Scanner{Prober: pb, Indexer: &fakeLister{}, Store: &fakeStore{}} + res := s.Scan(context.Background(), "https://eroticfeeldev.blob.core.windows.net/images") + if len(res) != 1 || res[0].Service != "blob_storage" { + t.Fatalf("unexpected result: %+v", res) + } + if pb.last.Container != "images" { + t.Fatalf("container hint not passed: %+v", pb.last) + } +} + +func TestScanViaCNAME(t *testing.T) { + pb := &fakeProber{state: model.ExposurePublicRead, listable: true} + st := &fakeStore{} + s := &Scanner{ + Prober: pb, + Indexer: &fakeLister{}, + Store: st, + Enricher: &fakeEnricher{chain: []string{"veteran1.nyc3.digitaloceanspaces.com"}}, + } + res := s.Scan(context.Background(), "cdn.mydomain.example") + if len(res) != 1 || res[0].Provider != "digitalocean" { + t.Fatalf("expected digitalocean via CNAME, got %+v", res) + } + if len(res[0].CNAMEChain) < 2 { + t.Fatalf("expected CNAME chain, got %+v", res[0].CNAMEChain) + } +} + +func TestScanUnknown(t *testing.T) { + s := &Scanner{Prober: &fakeProber{}, Store: &fakeStore{}} + res := s.Scan(context.Background(), "www.example.com") + if len(res) != 1 || res[0].Error == "" { + t.Fatalf("expected error result, got %+v", res) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 88c10a2..bc44de0 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -143,6 +143,50 @@ func nonNil(s []string) []string { return s } +// ListProbeTargets returns persisted assets to re-probe (for backfill). An +// empty exposure matches all. +func (s *Store) ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error) { + if limit <= 0 || limit > 100000 { + limit = 10000 + } + rows, err := s.pool.Query(ctx, ` + SELECT provider, service, provider_hostname, resource_identifier, region + FROM cloud_assets + WHERE ($1 = '' OR exposure_state = $1) + ORDER BY last_seen DESC + LIMIT $2`, exposure, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []model.ProbeTarget{} + for rows.Next() { + var t model.ProbeTarget + if err := rows.Scan(&t.Provider, &t.Service, &t.Hostname, &t.Resource, &t.Region); err != nil { + return nil, err + } + out = append(out, t) + } + return out, rows.Err() +} + +// EnsureAsset upserts a manually scanned asset (which may not have appeared in +// CT) and returns its id. +func (s *Store) EnsureAsset(ctx context.Context, c model.Candidate) (int64, error) { + var id int64 + err := s.pool.QueryRow(ctx, ` + INSERT INTO cloud_assets + (provider, service, hostname, provider_hostname, resource_identifier, + region, resource_specific, public_semantic, first_seen, last_seen) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now()) + ON CONFLICT (provider, service, provider_hostname) + DO UPDATE SET last_seen = now() + RETURNING id`, + c.Provider.String(), c.Service.String(), c.Original, c.Hostname, c.Resource, + c.Region, c.ResourceSpecific, c.PublicSemantic.String()).Scan(&id) + return id, err +} + // IndexEntry is one indexed resource item for the UI. type IndexEntry struct { ID int64 `json:"id"` @@ -267,12 +311,13 @@ func (s *Store) CountAssets(ctx context.Context, f AssetFilter) (int, error) { } // UpdateExposure records the outcome of an optional reachability probe. -func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState, detail string) error { +func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState, detail string, listable, objectReadable bool) error { _, err := s.pool.Exec(ctx, ` UPDATE cloud_assets - SET exposure_state = $4, exposure_detail = $5, probed_at = now() + SET exposure_state = $4, exposure_detail = $5, probed_at = now(), + listable = $6, object_readable = $7 WHERE provider = $1 AND service = $2 AND provider_hostname = $3`, - provider, service, providerHostname, state.String(), detail) + provider, service, providerHostname, state.String(), detail, listable, objectReadable) return err } @@ -288,6 +333,8 @@ type Asset struct { PublicSemantic string `json:"public_semantic"` Exposure string `json:"exposure_state"` ExposureDetail string `json:"exposure_detail"` + Listable bool `json:"listable"` + ObjectReadable bool `json:"object_readable"` FirstSeen time.Time `json:"first_seen"` LastSeen time.Time `json:"last_seen"` ProbedAt *time.Time `json:"probed_at"` @@ -327,7 +374,8 @@ func (s *Store) ListAssets(ctx context.Context, f AssetFilter) ([]Asset, error) } rows, err := s.pool.Query(ctx, ` SELECT id, provider, service, hostname, provider_hostname, resource_identifier, - region, public_semantic, exposure_state, exposure_detail, first_seen, last_seen, probed_at + region, public_semantic, exposure_state, exposure_detail, listable, object_readable, + first_seen, last_seen, probed_at FROM cloud_assets WHERE ($1 = '' OR provider = $1) AND ($2 = '' OR service = $2) @@ -345,7 +393,7 @@ func (s *Store) ListAssets(ctx context.Context, f AssetFilter) ([]Asset, error) var a Asset if err := rows.Scan(&a.ID, &a.Provider, &a.Service, &a.Hostname, &a.ProviderHost, &a.Resource, &a.Region, &a.PublicSemantic, &a.Exposure, &a.ExposureDetail, - &a.FirstSeen, &a.LastSeen, &a.ProbedAt); err != nil { + &a.Listable, &a.ObjectReadable, &a.FirstSeen, &a.LastSeen, &a.ProbedAt); err != nil { return nil, err } out = append(out, a) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index bd49cea..02e8a03 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -60,7 +60,7 @@ func TestIntegration(t *testing.T) { if err := s.WriteFindings(ctx, []pipeline.Finding{f}); err != nil { t.Fatal(err) } - if err := s.UpdateExposure(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, model.ExposureReachable, "test"); err != nil { + if err := s.UpdateExposure(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, model.ExposurePublicRead, "test", true, true); err != nil { t.Fatal(err) } if _, err := s.ListAssets(ctx, AssetFilter{Provider: "aws", Limit: 10}); err != nil { diff --git a/internal/webui/index.html b/internal/webui/index.html index 9228412..3809e9c 100644 --- a/internal/webui/index.html +++ b/internal/webui/index.html @@ -80,6 +80,16 @@ try{var t=localStorage.getItem('bucketeer-theme')||(matchMedia('(prefers-color-s .sort{margin-left:auto;display:flex;align-items:center;gap:8px;color:var(--text-secondary);font:400 14px/20px var(--font)} select{height:40px;border:1px solid var(--border-strong);border-radius:9999px;background:var(--canvas); color:var(--text-primary);font:400 14px/20px var(--font);padding:0 12px} + .scan{display:flex;gap:8px;margin-top:16px} + .scan input{flex:1;height:40px;border:1px solid var(--border-strong);border-radius:9999px;background:var(--surface-control); + color:var(--text-primary);font:400 14px/20px var(--font);padding:0 16px} + .scan input:focus-visible{outline:2px solid var(--focus);outline-offset:3px;background:var(--canvas)} + .scan button{min-height:40px;padding:0 20px;border-radius:9999px;border:1px solid var(--primary); + background:var(--primary);color:var(--on-primary);font:500 14px/20px var(--font);cursor:pointer} + .notice{margin-top:16px;padding:12px;border-radius:8px;background:var(--surface-subtle);border:1px solid var(--border)} + .notice-ok{background:var(--ok-bg);border-color:transparent} + .samples{margin-top:8px;border-top:1px solid var(--border);padding-top:8px} + .sample{color:var(--text-secondary);font-size:13px;overflow-wrap:anywhere} .summary{display:flex;gap:16px;flex-wrap:wrap;margin-top:24px;color:var(--text-muted);font:400 14px/20px var(--font)} .summary strong{color:var(--text-primary);font-weight:500} @@ -145,6 +155,14 @@ try{var t=localStorage.getItem('bucketeer-theme')||(matchMedia('(prefers-color-s +
+ + + +
+ +
@@ -256,6 +274,7 @@ function assetRow(a) { ${a.exposure_detail ? `

${esc(a.exposure_detail)}

` : ''}
${badge(a.provider, 'badge-cap')}${badge(a.service, 'badge-size')} + ${a.listable ? badge('listable', 'badge-ok') : (a.object_readable ? badge('object readable', 'badge-ok') : '')} ${a.resource_identifier ? badge(a.resource_identifier) : ''} ${a.public_semantic && a.public_semantic !== 'none' ? badge(a.public_semantic) : ''}
@@ -366,6 +385,40 @@ function syncURL() { $('asset-controls').hidden = state.view !== 'assets'; $('q').placeholder = state.view === 'assets' ? 'Search hostname, bucket, database…' : 'Search indexed contents…'; } +function scanResult(r) { + if (r.error) return `
${esc(r.host || r.target)}: ${esc(r.error)}
`; + const samples = (r.sample || []).map((s) => `
${esc(s)}
`).join(''); + const chain = r.cname_chain && r.cname_chain.length > 1 + ? `
chain: ${r.cname_chain.map(esc).join(' → ')}
` : ''; + return `
+
+ ${esc(r.host)}${exposureBadge(r.exposure)} +
+ ${r.detail ? `

${esc(r.detail)}

` : ''} +
${badge(r.provider, 'badge-cap')}${badge(r.service, 'badge-size')}${badge((r.entries || 0) + ' entries')}
+ ${chain}${samples ? `
${samples}
` : ''} +
`; +} +$('scanform').addEventListener('submit', async (e) => { + e.preventDefault(); + const target = $('scan').value.trim(); + if (!target) return; + const out = $('scanout'); + out.hidden = false; + out.innerHTML = '
'; + try { + const resp = await fetch('api/scan', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ target }), + }); + if (!resp.ok) throw new Error('HTTP ' + resp.status); + const rows = await resp.json() || []; + out.innerHTML = rows.map(scanResult).join(''); + if (rows.some((r) => r.entries > 0 || r.exposure === 'public_read')) reload(false); + } catch (err) { + out.innerHTML = `
Scan failed: ${esc(err.message)}
`; + } +}); + function setView(v) { state.view = v; syncURL(); reload(true); } $('nav-assets').onclick = (e) => { e.preventDefault(); setView('assets'); }; diff --git a/internal/webui/webui.go b/internal/webui/webui.go index 5adb14f..d79282f 100644 --- a/internal/webui/webui.go +++ b/internal/webui/webui.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "tea.karlcom.de/karl/bucketeer/internal/scan" "tea.karlcom.de/karl/bucketeer/internal/store" ) @@ -29,9 +30,17 @@ type Store interface { Stats(ctx context.Context) (store.Stats, error) } +// Scanner probes and indexes a user-supplied host/URL on demand, and can +// re-probe already-discovered assets. +type Scanner interface { + Scan(ctx context.Context, target string) []scan.Result + Backfill(ctx context.Context, exposure string, limit int) (scan.BackfillReport, error) +} + type Config struct { - User string // optional HTTP basic auth username - Pass string // optional HTTP basic auth password + User string // optional HTTP basic auth username + Pass string // optional HTTP basic auth password + Scanner Scanner } // New returns the UI handler. If Config.User is set, all routes require basic auth. @@ -84,6 +93,39 @@ func New(s Store, cfg Config) http.Handler { } writeJSON(w, map[string]any{"entries": entries, "total": total, "limit": limit, "offset": offset}) }) + if cfg.Scanner != nil { + mux.HandleFunc("/api/scan", func(w http.ResponseWriter, r *http.Request) { + target := strings.TrimSpace(r.URL.Query().Get("target")) + if r.Method == http.MethodPost { + var body struct { + Target string `json:"target"` + } + _ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body) + if body.Target != "" { + target = strings.TrimSpace(body.Target) + } + } + if target == "" || len(target) > 512 { + http.Error(w, "target required (<=512 chars)", http.StatusBadRequest) + return + } + writeJSON(w, cfg.Scanner.Scan(r.Context(), target)) + }) + mux.HandleFunc("/api/reprobe", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "POST only", http.StatusMethodNotAllowed) + return + } + q := r.URL.Query() + limit, _ := strconv.Atoi(q.Get("limit")) + rep, err := cfg.Scanner.Backfill(r.Context(), strings.TrimSpace(q.Get("exposure")), limit) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, rep) + }) + } mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) { st, err := s.Stats(r.Context()) if err != nil { diff --git a/migrations/0001_schema.sql b/migrations/0001_schema.sql index 8120e97..4b70245 100644 --- a/migrations/0001_schema.sql +++ b/migrations/0001_schema.sql @@ -23,6 +23,10 @@ CREATE INDEX IF NOT EXISTS cloud_assets_last_seen_idx ON cloud_assets (last_seen -- idempotent for existing databases). ALTER TABLE cloud_assets ADD COLUMN IF NOT EXISTS exposure_detail TEXT NOT NULL DEFAULT ''; ALTER TABLE cloud_assets ADD COLUMN IF NOT EXISTS probed_at TIMESTAMPTZ; +-- Readability capabilities, kept separate because they differ by provider +-- (e.g. Azure "Blob" = object readable only; R2 = object read, never listable). +ALTER TABLE cloud_assets ADD COLUMN IF NOT EXISTS listable BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE cloud_assets ADD COLUMN IF NOT EXISTS object_readable BOOLEAN NOT NULL DEFAULT FALSE; CREATE TABLE IF NOT EXISTS observations ( id BIGSERIAL PRIMARY KEY,