provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
This commit is contained in:
StasonJatham committed
1 parent
a366905a43
commit
62a7d2eb2e
23 files changed
+936
-165
No files matched your search
@@ -0,0 +1,214 @@
|
||||
// Package scan probes and indexes a user-supplied hostname or URL on demand.
|
||||
// This is how a specific public bucket that is not necessarily in the CT stream
|
||||
// gets verified and enumerated. It reuses the same classifier, prober and
|
||||
// indexer as the CT pipeline, so results are identical.
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/classify"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/dns"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/probe"
|
||||
)
|
||||
|
||||
// Store is the write subset the scanner needs.
|
||||
type Store interface {
|
||||
EnsureAsset(ctx context.Context, c model.Candidate) (int64, error)
|
||||
ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error)
|
||||
ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error)
|
||||
UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error
|
||||
}
|
||||
|
||||
// The dependencies are interfaces so the scanner is testable.
|
||||
type Prober interface {
|
||||
ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result
|
||||
}
|
||||
|
||||
type Lister interface {
|
||||
List(ctx context.Context, c model.Candidate) ([]model.Entry, error)
|
||||
}
|
||||
|
||||
type Enricher interface {
|
||||
Resolve(ctx context.Context, host string) (dns.Result, bool)
|
||||
}
|
||||
|
||||
type Scanner struct {
|
||||
Prober Prober
|
||||
Indexer Lister
|
||||
Enricher Enricher
|
||||
Store Store
|
||||
}
|
||||
|
||||
// Result is the outcome of scanning one resolved provider resource.
|
||||
type Result struct {
|
||||
Target string `json:"target"`
|
||||
Host string `json:"host"`
|
||||
Provider string `json:"provider"`
|
||||
Service string `json:"service"`
|
||||
Exposure string `json:"exposure"`
|
||||
Detail string `json:"detail"`
|
||||
Listable bool `json:"listable"`
|
||||
ObjectReadable bool `json:"object_readable"`
|
||||
Entries int `json:"entries"`
|
||||
Sample []string `json:"sample,omitempty"`
|
||||
CNAMEChain []string `json:"cname_chain,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Scan resolves the target (directly or through its CNAME chain), probes each
|
||||
// provider resource it maps to, and indexes readable ones.
|
||||
func (s *Scanner) Scan(ctx context.Context, target string) []Result {
|
||||
host, path := splitTarget(target)
|
||||
if host == "" {
|
||||
return []Result{{Target: target, Error: "empty target"}}
|
||||
}
|
||||
|
||||
if m, ok := classify.Classify(host); ok {
|
||||
c := candidateFor(host, host, m)
|
||||
if path != "" && c.Provider == model.ProviderAzure {
|
||||
c.Container = firstSegment(path)
|
||||
}
|
||||
return []Result{s.run(ctx, c, nil)}
|
||||
}
|
||||
|
||||
// Follow the CNAME chain and classify every target (custom domains pointing
|
||||
// at a bucket/CDN are the common case).
|
||||
if s.Enricher != nil {
|
||||
res, ok := s.Enricher.Resolve(ctx, host)
|
||||
if ok {
|
||||
var out []Result
|
||||
for _, t := range res.Chain[1:] {
|
||||
if m, matched := classify.Classify(t); matched && m.ResourceSpecific {
|
||||
out = append(out, s.run(ctx, candidateFor(t, host, m), res.Chain))
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []Result{{Target: target, Host: host, CNAMEChain: res.Chain,
|
||||
Error: "resolved, but no CNAME target matched a known provider pattern"}}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
return []Result{{Target: target, Host: host, Error: "host does not match a known provider pattern and has no provider CNAME"}}
|
||||
}
|
||||
|
||||
// BackfillReport summarizes a re-probe of already-discovered assets.
|
||||
type BackfillReport struct {
|
||||
Probed int `json:"probed"`
|
||||
Readable int `json:"readable"`
|
||||
Listable int `json:"listable"`
|
||||
Indexed int `json:"indexed"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// Backfill re-probes assets already in the database (optionally filtered by
|
||||
// exposure_state) with the current checks and indexes any that are readable.
|
||||
// This recovers findings discovered before a check was improved.
|
||||
func (s *Scanner) Backfill(ctx context.Context, exposure string, limit int) (BackfillReport, error) {
|
||||
targets, err := s.Store.ListProbeTargets(ctx, exposure, limit)
|
||||
if err != nil {
|
||||
return BackfillReport{}, err
|
||||
}
|
||||
var rep BackfillReport
|
||||
for _, t := range targets {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return rep, err
|
||||
}
|
||||
c := model.Candidate{
|
||||
Hostname: t.Hostname, Original: t.Hostname, Resource: t.Resource, Region: t.Region,
|
||||
Provider: model.ParseProvider(t.Provider), Service: model.ParseService(t.Service),
|
||||
Source: model.SourceManual,
|
||||
}
|
||||
pr := s.Prober.ProbeDetailed(ctx, c)
|
||||
rep.Probed++
|
||||
if pr.Exposure == model.ExposurePublicRead {
|
||||
rep.Readable++
|
||||
}
|
||||
if pr.Listable {
|
||||
rep.Listable++
|
||||
}
|
||||
if err := s.Store.UpdateExposure(ctx, t.Provider, t.Service, t.Hostname,
|
||||
pr.Exposure, pr.Detail, pr.Listable, pr.ObjectReadable); err != nil {
|
||||
rep.Errors++
|
||||
continue
|
||||
}
|
||||
if pr.Listable && s.Indexer != nil {
|
||||
entries, err := s.Indexer.List(ctx, c)
|
||||
if err == nil && len(entries) > 0 {
|
||||
if n, werr := s.Store.ReplaceEntries(ctx, t.Provider, t.Service, t.Hostname, entries); werr == nil {
|
||||
rep.Indexed += n
|
||||
} else {
|
||||
rep.Errors++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
func (s *Scanner) run(ctx context.Context, c model.Candidate, chain []string) Result {
|
||||
r := Result{Target: c.Original, Host: c.Hostname, Provider: c.Provider.String(),
|
||||
Service: c.Service.String(), CNAMEChain: chain}
|
||||
c.Source = model.SourceManual
|
||||
if _, err := s.Store.EnsureAsset(ctx, c); err != nil {
|
||||
r.Error = "store: " + err.Error()
|
||||
return r
|
||||
}
|
||||
pr := s.Prober.ProbeDetailed(ctx, c)
|
||||
r.Exposure, r.Detail = pr.Exposure.String(), pr.Detail
|
||||
r.Listable, r.ObjectReadable = pr.Listable, pr.ObjectReadable
|
||||
if pr.Listable && s.Indexer != nil {
|
||||
entries, err := s.Indexer.List(ctx, c)
|
||||
if err == nil && len(entries) > 0 {
|
||||
n, werr := s.Store.ReplaceEntries(ctx, c.Provider.String(), c.Service.String(), c.Hostname, entries)
|
||||
if werr != nil {
|
||||
r.Error = "index store: " + werr.Error()
|
||||
}
|
||||
r.Entries = n
|
||||
for i, e := range entries {
|
||||
if i >= 10 {
|
||||
break
|
||||
}
|
||||
r.Sample = append(r.Sample, e.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func candidateFor(host, original string, m model.Match) model.Candidate {
|
||||
return model.Candidate{
|
||||
Hostname: host, Original: original, Region: m.Region, Resource: m.Resource,
|
||||
Provider: m.Provider, Service: m.Service, PublicSemantic: m.PublicSemantic,
|
||||
ResourceSpecific: m.ResourceSpecific, Source: model.SourceManual,
|
||||
}
|
||||
}
|
||||
|
||||
// splitTarget accepts "host", "host/path" or a full URL and returns the host and
|
||||
// optional path (without a leading slash).
|
||||
func splitTarget(target string) (string, string) {
|
||||
target = strings.TrimSpace(target)
|
||||
if i := strings.Index(target, "://"); i >= 0 {
|
||||
target = target[i+3:]
|
||||
}
|
||||
host, path := target, ""
|
||||
if i := strings.IndexByte(target, '/'); i >= 0 {
|
||||
host, path = target[:i], target[i+1:]
|
||||
}
|
||||
host = strings.TrimSuffix(strings.ToLower(host), ".")
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
return host, path
|
||||
}
|
||||
|
||||
func firstSegment(path string) string {
|
||||
if i := strings.IndexByte(path, '/'); i >= 0 {
|
||||
return path[:i]
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/dns"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/probe"
|
||||
)
|
||||
|
||||
type fakeProber struct {
|
||||
last model.Candidate
|
||||
state model.ExposureState
|
||||
detail string
|
||||
listable bool
|
||||
obj bool
|
||||
}
|
||||
|
||||
func (f *fakeProber) ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result {
|
||||
f.last = c
|
||||
return probe.Result{Exposure: f.state, Detail: f.detail, Listable: f.listable, ObjectReadable: f.obj}
|
||||
}
|
||||
|
||||
type fakeLister struct{ entries []model.Entry }
|
||||
|
||||
func (f *fakeLister) List(ctx context.Context, c model.Candidate) ([]model.Entry, error) {
|
||||
return f.entries, nil
|
||||
}
|
||||
|
||||
type fakeEnricher struct{ chain []string }
|
||||
|
||||
func (f *fakeEnricher) Resolve(ctx context.Context, host string) (dns.Result, bool) {
|
||||
return dns.Result{Chain: append([]string{host}, f.chain...)}, true
|
||||
}
|
||||
|
||||
type fakeStore struct {
|
||||
ensured []model.Candidate
|
||||
entries int
|
||||
}
|
||||
|
||||
func (f *fakeStore) EnsureAsset(ctx context.Context, c model.Candidate) (int64, error) {
|
||||
f.ensured = append(f.ensured, c)
|
||||
return int64(len(f.ensured)), nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error) {
|
||||
f.entries = len(entries)
|
||||
return len(entries), nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestScanDirectReadable(t *testing.T) {
|
||||
pb := &fakeProber{state: model.ExposurePublicRead, detail: "s3 status=200", listable: true}
|
||||
st := &fakeStore{}
|
||||
s := &Scanner{Prober: pb, Indexer: &fakeLister{entries: []model.Entry{{Name: "a.txt"}, {Name: "b.txt"}}}, Store: st}
|
||||
res := s.Scan(context.Background(), "bucket-01.s3-eu-west-1.amazonaws.com")
|
||||
if len(res) != 1 || res[0].Exposure != "public_read" || res[0].Entries != 2 || len(res[0].Sample) != 2 {
|
||||
t.Fatalf("unexpected result: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAzureContainerPath(t *testing.T) {
|
||||
pb := &fakeProber{state: model.ExposurePublicRead, detail: "azure container 'images' is publicly listable", listable: true}
|
||||
s := &Scanner{Prober: pb, Indexer: &fakeLister{}, Store: &fakeStore{}}
|
||||
res := s.Scan(context.Background(), "https://eroticfeeldev.blob.core.windows.net/images")
|
||||
if len(res) != 1 || res[0].Service != "blob_storage" {
|
||||
t.Fatalf("unexpected result: %+v", res)
|
||||
}
|
||||
if pb.last.Container != "images" {
|
||||
t.Fatalf("container hint not passed: %+v", pb.last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanViaCNAME(t *testing.T) {
|
||||
pb := &fakeProber{state: model.ExposurePublicRead, listable: true}
|
||||
st := &fakeStore{}
|
||||
s := &Scanner{
|
||||
Prober: pb,
|
||||
Indexer: &fakeLister{},
|
||||
Store: st,
|
||||
Enricher: &fakeEnricher{chain: []string{"veteran1.nyc3.digitaloceanspaces.com"}},
|
||||
}
|
||||
res := s.Scan(context.Background(), "cdn.mydomain.example")
|
||||
if len(res) != 1 || res[0].Provider != "digitalocean" {
|
||||
t.Fatalf("expected digitalocean via CNAME, got %+v", res)
|
||||
}
|
||||
if len(res[0].CNAMEChain) < 2 {
|
||||
t.Fatalf("expected CNAME chain, got %+v", res[0].CNAMEChain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanUnknown(t *testing.T) {
|
||||
s := &Scanner{Prober: &fakeProber{}, Store: &fakeStore{}}
|
||||
res := s.Scan(context.Background(), "www.example.com")
|
||||
if len(res) != 1 || res[0].Error == "" {
|
||||
t.Fatalf("expected error result, got %+v", res)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user