// Package scan probes and indexes a user-supplied hostname or URL on demand. // This is how a specific public bucket that is not necessarily in the CT stream // gets verified and enumerated. It reuses the same classifier, prober and // indexer as the CT pipeline, so results are identical. package scan import ( "context" "net" "strings" "sync" "sync/atomic" "tea.karlcom.de/karl/bucketeer/internal/classify" "tea.karlcom.de/karl/bucketeer/internal/dns" "tea.karlcom.de/karl/bucketeer/internal/model" "tea.karlcom.de/karl/bucketeer/internal/probe" ) // Store is the write subset the scanner needs. type Store interface { EnsureAsset(ctx context.Context, c model.Candidate) (int64, error) ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error) ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error) UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error } // The dependencies are interfaces so the scanner is testable. type Prober interface { ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result } type Lister interface { List(ctx context.Context, c model.Candidate) ([]model.Entry, error) } type Enricher interface { Resolve(ctx context.Context, host string) (dns.Result, bool) } type Scanner struct { Prober Prober Indexer Lister Enricher Enricher Store Store BackfillWorkers int } // Result is the outcome of scanning one resolved provider resource. type Result struct { Target string `json:"target"` Host string `json:"host"` Provider string `json:"provider"` Service string `json:"service"` Exposure string `json:"exposure"` Detail string `json:"detail"` Listable bool `json:"listable"` ObjectReadable bool `json:"object_readable"` Entries int `json:"entries"` Sample []string `json:"sample,omitempty"` CNAMEChain []string `json:"cname_chain,omitempty"` Error string `json:"error,omitempty"` } // Scan resolves the target (directly or through its CNAME chain), probes each // provider resource it maps to, and indexes readable ones. func (s *Scanner) Scan(ctx context.Context, target string) []Result { host, path := splitTarget(target) if host == "" { return []Result{{Target: target, Error: "empty target"}} } if m, ok := classify.Classify(host); ok { c := candidateFor(host, host, m) if path != "" && c.Provider == model.ProviderAzure { c.Container = firstSegment(path) } return []Result{s.run(ctx, c, nil)} } // Follow the CNAME chain and classify every target (custom domains pointing // at a bucket/CDN are the common case). if s.Enricher != nil { res, ok := s.Enricher.Resolve(ctx, host) if ok { var out []Result for _, t := range res.Chain[1:] { if m, matched := classify.Classify(t); matched && m.ResourceSpecific { out = append(out, s.run(ctx, candidateFor(t, host, m), res.Chain)) } } if len(out) == 0 { return []Result{{Target: target, Host: host, CNAMEChain: res.Chain, Error: "resolved, but no CNAME target matched a known provider pattern"}} } return out } } return []Result{{Target: target, Host: host, Error: "host does not match a known provider pattern and has no provider CNAME"}} } // BackfillReport summarizes a re-probe of already-discovered assets. type BackfillReport struct { Probed int `json:"probed"` Readable int `json:"readable"` Listable int `json:"listable"` Indexed int `json:"indexed"` Errors int `json:"errors"` } // Backfill re-probes assets already in the database (optionally filtered by // exposure_state) with the current checks and indexes any that are readable. // This is how already-discovered "reachable" endpoints actually get verified. // It runs a bounded worker pool so it stays interactive. func (s *Scanner) Backfill(ctx context.Context, exposure string, limit int) (BackfillReport, error) { targets, err := s.Store.ListProbeTargets(ctx, exposure, limit) if err != nil { return BackfillReport{}, err } workers := s.BackfillWorkers if workers <= 0 { workers = 32 } var ( probed, readable, listable, indexed, errs atomic.Int64 wg sync.WaitGroup jobs = make(chan model.ProbeTarget) ) for i := 0; i < workers; i++ { wg.Add(1) go func() { defer wg.Done() for t := range jobs { if ctx.Err() != nil { return } s.probeTarget(ctx, t, &probed, &readable, &listable, &indexed, &errs) } }() } for _, t := range targets { select { case jobs <- t: case <-ctx.Done(): close(jobs) wg.Wait() return s.report(&probed, &readable, &listable, &indexed, &errs), ctx.Err() } } close(jobs) wg.Wait() return s.report(&probed, &readable, &listable, &indexed, &errs), nil } func (s *Scanner) probeTarget(ctx context.Context, t model.ProbeTarget, probed, readable, listable, indexed, errs *atomic.Int64) { c := model.Candidate{ Hostname: t.Hostname, Original: t.Hostname, Resource: t.Resource, Region: t.Region, Provider: model.ParseProvider(t.Provider), Service: model.ParseService(t.Service), Source: model.SourceManual, } pr := s.Prober.ProbeDetailed(ctx, c) probed.Add(1) if pr.Exposure == model.ExposurePublicRead { readable.Add(1) } if pr.Listable { listable.Add(1) } if err := s.Store.UpdateExposure(ctx, t.Provider, t.Service, t.Hostname, pr.Exposure, pr.Detail, pr.Listable, pr.ObjectReadable); err != nil { errs.Add(1) return } if pr.Listable && s.Indexer != nil { entries, err := s.Indexer.List(ctx, c) if err == nil && len(entries) > 0 { if n, werr := s.Store.ReplaceEntries(ctx, t.Provider, t.Service, t.Hostname, entries); werr == nil { indexed.Add(int64(n)) } else { errs.Add(1) } } } } func (s *Scanner) report(probed, readable, listable, indexed, errs *atomic.Int64) BackfillReport { return BackfillReport{ Probed: int(probed.Load()), Readable: int(readable.Load()), Listable: int(listable.Load()), Indexed: int(indexed.Load()), Errors: int(errs.Load()), } } func (s *Scanner) run(ctx context.Context, c model.Candidate, chain []string) Result { r := Result{Target: c.Original, Host: c.Hostname, Provider: c.Provider.String(), Service: c.Service.String(), CNAMEChain: chain} c.Source = model.SourceManual if _, err := s.Store.EnsureAsset(ctx, c); err != nil { r.Error = "store: " + err.Error() return r } pr := s.Prober.ProbeDetailed(ctx, c) r.Exposure, r.Detail = pr.Exposure.String(), pr.Detail r.Listable, r.ObjectReadable = pr.Listable, pr.ObjectReadable if err := s.Store.UpdateExposure(ctx, c.Provider.String(), c.Service.String(), c.Hostname, pr.Exposure, pr.Detail, pr.Listable, pr.ObjectReadable); err != nil { r.Error = "store: " + err.Error() } if pr.Listable && s.Indexer != nil { entries, err := s.Indexer.List(ctx, c) if err == nil && len(entries) > 0 { n, werr := s.Store.ReplaceEntries(ctx, c.Provider.String(), c.Service.String(), c.Hostname, entries) if werr != nil { r.Error = "index store: " + werr.Error() } r.Entries = n for i, e := range entries { if i >= 10 { break } r.Sample = append(r.Sample, e.Name) } } } return r } func candidateFor(host, original string, m model.Match) model.Candidate { return model.Candidate{ Hostname: host, Original: original, Region: m.Region, Resource: m.Resource, Provider: m.Provider, Service: m.Service, PublicSemantic: m.PublicSemantic, ResourceSpecific: m.ResourceSpecific, Source: model.SourceManual, } } // splitTarget accepts "host", "host/path" or a full URL and returns the host and // optional path (without a leading slash). func splitTarget(target string) (string, string) { target = strings.TrimSpace(target) if i := strings.Index(target, "://"); i >= 0 { target = target[i+3:] } host, path := target, "" if i := strings.IndexByte(target, '/'); i >= 0 { host, path = target[:i], target[i+1:] } host = strings.TrimSuffix(strings.ToLower(host), ".") if h, _, err := net.SplitHostPort(host); err == nil { host = h } return host, path } func firstSegment(path string) string { if i := strings.IndexByte(path, '/'); i >= 0 { return path[:i] } return path }