bucketeer/internal/scan/scanner.go

// Package scan probes and indexes a user-supplied hostname or URL on demand.
// This is how a specific public bucket that is not necessarily in the CT stream
// gets verified and enumerated. It reuses the same classifier, prober and
// indexer as the CT pipeline, so results are identical.
package scan

import (
	"context"
	"net"
	"strings"
	"sync"
	"sync/atomic"

	"tea.karlcom.de/karl/bucketeer/internal/classify"
	"tea.karlcom.de/karl/bucketeer/internal/dns"
	"tea.karlcom.de/karl/bucketeer/internal/model"
	"tea.karlcom.de/karl/bucketeer/internal/probe"
)

// Store is the write subset the scanner needs.
type Store interface {
	EnsureAsset(ctx context.Context, c model.Candidate) (int64, error)
	ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error)
	ListProbeTargets(ctx context.Context, exposure string, limit int) ([]model.ProbeTarget, error)
	UpdateExposure(ctx context.Context, provider, service, host string, state model.ExposureState, detail string, listable, objectReadable bool) error
}

// The dependencies are interfaces so the scanner is testable.
type Prober interface {
	ProbeDetailed(ctx context.Context, c model.Candidate) probe.Result
}

type Lister interface {
	List(ctx context.Context, c model.Candidate) ([]model.Entry, error)
}

type Enricher interface {
	Resolve(ctx context.Context, host string) (dns.Result, bool)
}

type Scanner struct {
	Prober          Prober
	Indexer         Lister
	Enricher        Enricher
	Store           Store
	BackfillWorkers int
}

// Result is the outcome of scanning one resolved provider resource.
type Result struct {
	Target         string   `json:"target"`
	Host           string   `json:"host"`
	Provider       string   `json:"provider"`
	Service        string   `json:"service"`
	Exposure       string   `json:"exposure"`
	Detail         string   `json:"detail"`
	Listable       bool     `json:"listable"`
	ObjectReadable bool     `json:"object_readable"`
	Entries        int      `json:"entries"`
	Sample         []string `json:"sample,omitempty"`
	CNAMEChain     []string `json:"cname_chain,omitempty"`
	Error          string   `json:"error,omitempty"`
}

// Scan resolves the target (directly or through its CNAME chain), probes each
// provider resource it maps to, and indexes readable ones.
func (s *Scanner) Scan(ctx context.Context, target string) []Result {
	host, path := splitTarget(target)
	if host == "" {
		return []Result{{Target: target, Error: "empty target"}}
	}

	if m, ok := classify.Classify(host); ok {
		c := candidateFor(host, host, m)
		if path != "" && c.Provider == model.ProviderAzure {
			c.Container = firstSegment(path)
		}
		return []Result{s.run(ctx, c, nil)}
	}

	// Follow the CNAME chain and classify every target (custom domains pointing
	// at a bucket/CDN are the common case).
	if s.Enricher != nil {
		res, ok := s.Enricher.Resolve(ctx, host)
		if ok {
			var out []Result
			for _, t := range res.Chain[1:] {
				if m, matched := classify.Classify(t); matched && m.ResourceSpecific {
					out = append(out, s.run(ctx, candidateFor(t, host, m), res.Chain))
				}
			}
			if len(out) == 0 {
				return []Result{{Target: target, Host: host, CNAMEChain: res.Chain,
					Error: "resolved, but no CNAME target matched a known provider pattern"}}
			}
			return out
		}
	}
	return []Result{{Target: target, Host: host, Error: "host does not match a known provider pattern and has no provider CNAME"}}
}

// BackfillReport summarizes a re-probe of already-discovered assets.
type BackfillReport struct {
	Probed   int `json:"probed"`
	Readable int `json:"readable"`
	Listable int `json:"listable"`
	Indexed  int `json:"indexed"`
	Errors   int `json:"errors"`
}

// Backfill re-probes assets already in the database (optionally filtered by
// exposure_state) with the current checks and indexes any that are readable.
// This is how already-discovered "reachable" endpoints actually get verified.
// It runs a bounded worker pool so it stays interactive.
func (s *Scanner) Backfill(ctx context.Context, exposure string, limit int) (BackfillReport, error) {
	targets, err := s.Store.ListProbeTargets(ctx, exposure, limit)
	if err != nil {
		return BackfillReport{}, err
	}
	workers := s.BackfillWorkers
	if workers <= 0 {
		workers = 32
	}
	var (
		probed, readable, listable, indexed, errs atomic.Int64
		wg                                        sync.WaitGroup
		jobs                                      = make(chan model.ProbeTarget)
	)
	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for t := range jobs {
				if ctx.Err() != nil {
					return
				}
				s.probeTarget(ctx, t, &probed, &readable, &listable, &indexed, &errs)
			}
		}()
	}
	for _, t := range targets {
		select {
		case jobs <- t:
		case <-ctx.Done():
			close(jobs)
			wg.Wait()
			return s.report(&probed, &readable, &listable, &indexed, &errs), ctx.Err()
		}
	}
	close(jobs)
	wg.Wait()
	return s.report(&probed, &readable, &listable, &indexed, &errs), nil
}

func (s *Scanner) probeTarget(ctx context.Context, t model.ProbeTarget, probed, readable, listable, indexed, errs *atomic.Int64) {
	c := model.Candidate{
		Hostname: t.Hostname, Original: t.Hostname, Resource: t.Resource, Region: t.Region,
		Provider: model.ParseProvider(t.Provider), Service: model.ParseService(t.Service),
		Source: model.SourceManual,
	}
	pr := s.Prober.ProbeDetailed(ctx, c)
	probed.Add(1)
	if pr.Exposure == model.ExposurePublicRead {
		readable.Add(1)
	}
	if pr.Listable {
		listable.Add(1)
	}
	if err := s.Store.UpdateExposure(ctx, t.Provider, t.Service, t.Hostname,
		pr.Exposure, pr.Detail, pr.Listable, pr.ObjectReadable); err != nil {
		errs.Add(1)
		return
	}
	if pr.Listable && s.Indexer != nil {
		entries, err := s.Indexer.List(ctx, c)
		if err == nil && len(entries) > 0 {
			if n, werr := s.Store.ReplaceEntries(ctx, t.Provider, t.Service, t.Hostname, entries); werr == nil {
				indexed.Add(int64(n))
			} else {
				errs.Add(1)
			}
		}
	}
}

func (s *Scanner) report(probed, readable, listable, indexed, errs *atomic.Int64) BackfillReport {
	return BackfillReport{
		Probed: int(probed.Load()), Readable: int(readable.Load()),
		Listable: int(listable.Load()), Indexed: int(indexed.Load()), Errors: int(errs.Load()),
	}
}

func (s *Scanner) run(ctx context.Context, c model.Candidate, chain []string) Result {
	r := Result{Target: c.Original, Host: c.Hostname, Provider: c.Provider.String(),
		Service: c.Service.String(), CNAMEChain: chain}
	c.Source = model.SourceManual
	if _, err := s.Store.EnsureAsset(ctx, c); err != nil {
		r.Error = "store: " + err.Error()
		return r
	}
	pr := s.Prober.ProbeDetailed(ctx, c)
	r.Exposure, r.Detail = pr.Exposure.String(), pr.Detail
	r.Listable, r.ObjectReadable = pr.Listable, pr.ObjectReadable
	if err := s.Store.UpdateExposure(ctx, c.Provider.String(), c.Service.String(), c.Hostname,
		pr.Exposure, pr.Detail, pr.Listable, pr.ObjectReadable); err != nil {
		r.Error = "store: " + err.Error()
	}
	if pr.Listable && s.Indexer != nil {
		entries, err := s.Indexer.List(ctx, c)
		if err == nil && len(entries) > 0 {
			n, werr := s.Store.ReplaceEntries(ctx, c.Provider.String(), c.Service.String(), c.Hostname, entries)
			if werr != nil {
				r.Error = "index store: " + werr.Error()
			}
			r.Entries = n
			for i, e := range entries {
				if i >= 10 {
					break
				}
				r.Sample = append(r.Sample, e.Name)
			}
		}
	}
	return r
}

func candidateFor(host, original string, m model.Match) model.Candidate {
	return model.Candidate{
		Hostname: host, Original: original, Region: m.Region, Resource: m.Resource,
		Provider: m.Provider, Service: m.Service, PublicSemantic: m.PublicSemantic,
		ResourceSpecific: m.ResourceSpecific, Source: model.SourceManual,
	}
}

// splitTarget accepts "host", "host/path" or a full URL and returns the host and
// optional path (without a leading slash).
func splitTarget(target string) (string, string) {
	target = strings.TrimSpace(target)
	if i := strings.Index(target, "://"); i >= 0 {
		target = target[i+3:]
	}
	host, path := target, ""
	if i := strings.IndexByte(target, '/'); i >= 0 {
		host, path = target[:i], target[i+1:]
	}
	host = strings.TrimSuffix(strings.ToLower(host), ".")
	if h, _, err := net.SplitHostPort(host); err == nil {
		host = h
	}
	return host, path
}

func firstSegment(path string) string {
	if i := strings.IndexByte(path, '/'); i >= 0 {
		return path[:i]
	}
	return path
}