over-engineering audit: remove dead code, speculative config, duplicate queues

- delete embedded community resolver snapshot (6,760 lines) + EmbeddedServers +
  DNS_SERVERS=fresh; curated resolvers are the default. Simplify Warm (no
  sampling).
- one generic internal/sample.Queue[T] replaces three near-identical
  random-sample queues (probe, pipeline, discover).
- delete unreachable index.listOpenSearch (OpenSearch is never listable).
- delete classifyCloudFront (its match is always discarded, !ResourceSpecific).
- delete probe.Probe wrapper + Describe; tests use ProbeDetailed.
- delete store.Pool(), model.SourceDNSAlias, discover.QueueLen.
- drop dead config/no-op knobs: pipeline DirectWorkers, discover MaxPerDomain,
  discover S3/GCS/DO flags, BUCKET_SCAN_DO.
~-7,000 lines (mostly the resolver snapshot), 0 deps removed, tests green.

.env.example

@@ -11,7 +11,6 @@ DATABASE_URL=postgres://ctcloud:change-me-strong@postgres:5432/ctcloud?sslmode=d
 # DATABASE_URL=postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full
 
 # DNS_SERVERS unset = ~20 curated public resolvers. Options:
-#   DNS_SERVERS=fresh              curated + compiled-in Fresh-Resolvers (sampled)
 #   DNS_SERVERS=system             use the OS resolver
 #   DNS_SERVERS=1.1.1.1,8.8.8.8    your own comma-separated list
 # DNS_SERVERS=system
@@ -34,4 +33,3 @@ BUCKET_SCAN_ENABLED=true
 BUCKET_SCAN_QPS=50
 BUCKET_SCAN_WORKERS=16
 BUCKET_SCAN_QUEUE=65536
-BUCKET_SCAN_DO=false

README.md

@@ -131,7 +131,6 @@ Environment variables only; no config files, no runtime pattern files.
 | `BUCKET_SCAN_QPS` | `50` | global cap on outbound discovery requests |
 | `BUCKET_SCAN_WORKERS` | `16` | discovery probe workers |
 | `BUCKET_SCAN_QUEUE` | `65536` | sampled domain backlog |
-| `BUCKET_SCAN_DO` | `false` | also test DigitalOcean Spaces regions |
 | `DEDUP_TTL` | `1h` | hostname/finding dedup window |
 | `METRICS_ADDR` | `:9090` | metrics/health/UI listener |
 | `UI_USER` / `UI_PASS` | *(unset)* | optional HTTP basic auth for the web UI |
@@ -318,11 +317,10 @@ engine that:
 
 1. takes the names that actually occur in the hostname — the subdomain label
    and the registrable-domain label (`assets.example.com` → `assets`,
-   `example`). No names are fabricated, permuted, or drawn from a wordlist;
-2. tests anonymous listing per provider —
-   `https://<name>.s3.amazonaws.com/?list-type=2` (AWS),
-   `https://<name>.storage.googleapis.com` (GCS),
-   `https://<name>.<region>.digitaloceanspaces.com` (DO, optional);
+   `example`). No names are fabricated or drawn from a wordlist;
+2. tests anonymous listing —
+   `https://<name>.s3.amazonaws.com/?list-type=2` (AWS) and
+   `https://<name>.storage.googleapis.com` (GCS);
 3. indexes every bucket that returns 200 (keys/sizes/dates only).
 
 It is aggressively bounded: a global `BUCKET_SCAN_QPS` limit, a 12h per-name
@@ -343,22 +341,7 @@ healthy resolver and fails over to at most two others; each attempt is bounded
 answers (NXDOMAIN) return immediately rather than being retried on every server,
 so negative lookups do not multiply DNS traffic.
 
-Set `DNS_SERVERS=fresh` to add a compiled-in snapshot of
-[proabiral/Fresh-Resolvers](https://github.com/proabiral/Fresh-Resolvers)
-(6,760 entries) on top of the curated set. It validates a random sample of 256 at
-startup and cools down the rest on failure. Refresh the snapshot with:
-
-```sh
-curl -sL https://raw.githubusercontent.com/proabiral/Fresh-Resolvers/master/resolvers.txt \
-  -o internal/dns/resolvers.txt
-```
-
-> Probing or using thousands of community resolvers at once can saturate the
-> container's own DNS and break unrelated lookups, which is why `fresh` is
-> opt-in and sampled. These resolvers are also untrusted: they can observe and,
-> in principle, forge DNS answers. That only affects enrichment (which endpoints
-> are looked up), never the classifier. For sensitive deployments set
-> `DNS_SERVERS` to your own resolvers, or `DNS_SERVERS=system`.
+> For sensitive deployments set `DNS_SERVERS` to your own resolvers, or `DNS_SERVERS=system`.
 
 > These community resolvers are untrusted: they can observe and, in principle,
 > forge DNS answers. That only affects enrichment (which endpoints are looked

cmd/ctcloud/main.go

@@ -108,15 +108,9 @@ func run() error {
 		// of costing a timeout on every random pick.
 		resolver = dns.NewServerPool(dns.CuratedServers())
 		warmCtx, cancel := context.WithTimeout(ctx, 8*time.Second)
-		healthy := resolver.Warm(warmCtx, "example.com", 16, 64)
+		healthy := resolver.Warm(warmCtx, "example.com", 16)
 		cancel()
 		slog.Info("dns resolvers configured", "count", resolver.Len(), "healthy", healthy, "set", "curated")
-	case len(dnsServers) == 1 && dnsServers[0] == "fresh":
-		resolver = dns.NewServerPool(append(dns.CuratedServers(), dns.EmbeddedServers()...))
-		warmCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
-		healthy := resolver.Warm(warmCtx, "example.com", 32, 256)
-		cancel()
-		slog.Info("dns resolvers warmed", "total", resolver.Len(), "healthy", healthy, "set", "fresh")
 	default:
 		resolver = dns.NewServerPool(dnsServers)
 		slog.Info("dns resolvers configured", "count", resolver.Len(), "set", "custom")
@@ -150,9 +144,7 @@ func run() error {
 	if cfg.BucketScanEnabled {
 		discoverer = discover.New(discover.Config{
 			QPS: cfg.BucketScanQPS, Workers: cfg.BucketScanWorkers, Queue: cfg.BucketScanQueue,
-			S3: true, GCS: true, DOSpaces: cfg.BucketScanDO,
-			DORegions: []string{"nyc3", "sfo3", "fra1", "ams3", "sgp1"},
-			Indexer:   indexer, Metrics: m,
+			Indexer: indexer, Metrics: m,
 			OnOpen: func(uctx context.Context, c model.Candidate, detail string, entries []model.Entry) {
 				if _, err := st.EnsureAsset(uctx, c); err != nil {
 					return
@@ -168,7 +160,7 @@ func run() error {
 		})
 		discoverer.Start(ctx)
 		slog.Info("bucket name discovery enabled", "event", "startup",
-			"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers, "do_spaces", cfg.BucketScanDO)
+			"qps", cfg.BucketScanQPS, "workers", cfg.BucketScanWorkers)
 	}
 
 	var prober *probe.Prober

internal/classify/aws.go

@@ -17,25 +17,6 @@ func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource,
 	}
 }
 
-// classifyCloudFront matches <distribution>.cloudfront.net. Custom domains
-// almost always CNAME to a CloudFront distribution, so this is how CDN-hosted
-// static sites get discovered via DNS enrichment.
-func classifyCloudFront(s *scanner) (model.Match, bool) {
-	n := s.n
-	if n != 3 || s.label(n-1) != "net" || s.label(n-2) != "cloudfront" {
-		return model.Match{}, false
-	}
-	// CloudFront distributions are CDN endpoints, not buckets or databases, so
-	// they are identified but not stored as findings.
-	return model.Match{
-		Provider:         model.ProviderAWS,
-		Service:          model.ServiceWebApp,
-		ResourceSpecific: false,
-		PublicSemantic:   model.PublicSemanticPublicEndpoint,
-		Resource:         s.label(0),
-	}, true
-}
-
 func classifyAWS(s *scanner) (model.Match, bool) {
 	n := s.n
 	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {

internal/classify/classify.go

@@ -114,8 +114,6 @@ func Classify(host string) (model.Match, bool) {
 	switch {
 	case strings.HasSuffix(host, ".amazonaws.com"):
 		return classifyAWS(&s)
-	case strings.HasSuffix(host, ".cloudfront.net"):
-		return classifyCloudFront(&s)
 	case strings.HasSuffix(host, ".core.windows.net"):
 		return classifyAzureCore(&s)
 	case strings.HasSuffix(host, ".database.windows.net"):

internal/classify/classify_test.go

@@ -93,7 +93,6 @@ func TestClassifyMatches(t *testing.T) {
 		{"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}},
 		{"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}},
 		{"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}},
-		{"d111111abcdef8.cloudfront.net", want{model.ProviderAWS, model.ServiceWebApp, "d111111abcdef8", "", false, model.PublicSemanticPublicEndpoint}},
 		// Cloudflare
 		{"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}},
 		{"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}},

internal/config/config.go

@@ -43,7 +43,6 @@ type Config struct {
 	BucketScanQPS     int
 	BucketScanWorkers int
 	BucketScanQueue   int
-	BucketScanDO      bool
 
 	MetricsAddr   string
 	UIUser        string
@@ -89,7 +88,6 @@ func Load() (*Config, error) {
 		BucketScanQPS:     envInt("BUCKET_SCAN_QPS", 50),
 		BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
 		BucketScanQueue:   envInt("BUCKET_SCAN_QUEUE", 65536),
-		BucketScanDO:      envBool("BUCKET_SCAN_DO", false),
 		MetricsAddr:       envString("METRICS_ADDR", ":9090"),
 		UIUser:            envString("UI_USER", ""),
 		UIPass:            envString("UI_PASS", ""),

internal/discover/discover.go

@@ -12,7 +12,6 @@ package discover
 import (
 	"context"
 	"io"
-	"math/rand/v2"
 	"net/http"
 	"strings"
 	"sync"
@@ -22,18 +21,13 @@ import (
 	"tea.karlcom.de/karl/bucketeer/internal/index"
 	"tea.karlcom.de/karl/bucketeer/internal/metrics"
 	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/sample"
 )
 
 type Config struct {
-	QPS          int // global outbound request rate
-	Workers      int // concurrent candidate probes
-	Queue        int // sampled domain backlog
-	S3           bool
-	GCS          bool
-	DOSpaces     bool
-	DORegions    []string
-	MaxPerDomain int // cap candidates generated per domain
-
+	QPS     int // global outbound request rate
+	Workers int // concurrent candidate probes
+	Queue   int // sampled domain backlog
 	Indexer *index.Indexer
 	Metrics *metrics.Metrics
 	// OnOpen is called for every anonymously listable bucket with its metadata
@@ -44,7 +38,7 @@ type Config struct {
 type Discoverer struct {
 	cfg    Config
 	client *http.Client
-	q      *domainQueue
+	q      *sample.Queue[string]
 	tokens chan struct{}
 	seen   *dns.TTLCache[bool] // candidate names already tested
 	stop   chan struct{}
@@ -61,13 +55,10 @@ func New(cfg Config) *Discoverer {
 	if cfg.Queue <= 0 {
 		cfg.Queue = 65536
 	}
-	if cfg.MaxPerDomain <= 0 {
-		cfg.MaxPerDomain = 48
-	}
 	return &Discoverer{
 		cfg:    cfg,
 		client: &http.Client{Timeout: 6 * time.Second},
-		q:      newDomainQueue(cfg.Queue),
+		q:      sample.New[string](cfg.Queue),
 		tokens: make(chan struct{}, max(cfg.QPS, 1)),
 		seen:   dns.NewTTLCache[bool](1<<20, 64),
 		stop:   make(chan struct{}),
@@ -77,7 +68,7 @@ func New(cfg Config) *Discoverer {
 // Submit offers a hostname to the discovery engine. Duplicate registrable
 // domains and already-tested candidate names are dropped cheaply.
 func (d *Discoverer) Submit(host string) {
-	d.q.push(host)
+	d.q.Push(host)
 }
 
 func (d *Discoverer) Start(ctx context.Context) {
@@ -92,7 +83,6 @@ func (d *Discoverer) Close() { close(d.stop) }
 func (d *Discoverer) Wait()  { d.wg.Wait() }
 
 // QueueLen reports the sampled domain backlog.
-func (d *Discoverer) QueueLen() int { return d.q.len() }
 
 func (d *Discoverer) refill(ctx context.Context) {
 	period := time.Second / time.Duration(d.cfg.QPS)
@@ -135,10 +125,10 @@ func (d *Discoverer) worker(ctx context.Context) {
 			return
 		case <-d.stop:
 			return
-		case <-d.q.notify:
+		case <-d.q.Notify():
 		}
 		for {
-			host, ok := d.q.pop()
+			host, ok := d.q.Pop()
 			if !ok {
 				break
 			}
@@ -148,7 +138,7 @@ func (d *Discoverer) worker(ctx context.Context) {
 }
 
 func (d *Discoverer) scanDomain(ctx context.Context, host string) {
-	for _, name := range candidates(host, d.cfg.MaxPerDomain) {
+	for _, name := range candidates(host) {
 		if ctx.Err() != nil {
 			return
 		}
@@ -183,35 +173,21 @@ type Target struct {
 }
 
 func (d *Discoverer) targets(name string) []Target {
-	var out []Target
-	if d.cfg.S3 {
-		out = append(out, Target{
+	return []Target{
+		{
 			Name: name, Provider: model.ProviderAWS, Service: model.ServiceS3,
 			Hostname: name + ".s3.amazonaws.com",
 			URL:      "https://" + name + ".s3.amazonaws.com/?list-type=2&max-keys=1",
-		})
-	}
-	if d.cfg.GCS {
-		// Use the virtual-host form so each bucket has a unique hostname (the
-		// path form makes every bucket share storage.googleapis.com and collide
-		// on the cloud_assets unique key).
-		out = append(out, Target{
+		},
+		{
+			// Virtual-host form so each bucket has a unique hostname (the path
+			// form makes every bucket share storage.googleapis.com and collide
+			// on the cloud_assets unique key).
 			Name: name, Provider: model.ProviderGCP, Service: model.ServiceObjectStorage,
 			Hostname: name + ".storage.googleapis.com",
 			URL:      "https://storage.googleapis.com/" + name + "?max-keys=1",
-		})
+		},
 	}
-	if d.cfg.DOSpaces {
-		for _, region := range d.cfg.DORegions {
-			host := name + "." + region + ".digitaloceanspaces.com"
-			out = append(out, Target{
-				Name: name, Provider: model.ProviderDigitalOcean, Service: model.ServiceObjectStorage,
-				Hostname: host, Region: region,
-				URL: "https://" + host + "/?list-type=2&max-keys=1",
-			})
-		}
-	}
-	return out
 }
 
 func (d *Discoverer) test(ctx context.Context, t Target) {
@@ -286,13 +262,7 @@ var skipLabel = map[string]bool{
 	"ns": true, "mx": true, "localhost": true,
 }
 
-func candidates(host string, max int) []string {
-	out := baseLabels(host)
-	if max > 0 && len(out) > max {
-		out = out[:max]
-	}
-	return out
-}
+func candidates(host string) []string { return baseLabels(host) }
 
 func validLabel(s string) bool {
 	if len(s) < 3 || len(s) > 40 {
@@ -319,51 +289,3 @@ func validBucketName(s string) bool {
 	}
 	return s[0] != '-' && s[0] != '.' && s[len(s)-1] != '-' && s[len(s)-1] != '.'
 }
-
-// --- sampled domain queue ---
-
-type domainQueue struct {
-	mu     sync.Mutex
-	items  []string
-	cap    int
-	notify chan struct{}
-}
-
-func newDomainQueue(capacity int) *domainQueue {
-	return &domainQueue{cap: capacity, notify: make(chan struct{}, 1)}
-}
-
-func (q *domainQueue) push(s string) {
-	q.mu.Lock()
-	if len(q.items) < q.cap {
-		q.items = append(q.items, s)
-	} else {
-		q.items[rand.IntN(len(q.items))] = s
-	}
-	q.mu.Unlock()
-	select {
-	case q.notify <- struct{}{}:
-	default:
-	}
-}
-
-func (q *domainQueue) pop() (string, bool) {
-	q.mu.Lock()
-	n := len(q.items)
-	if n == 0 {
-		q.mu.Unlock()
-		return "", false
-	}
-	i := rand.IntN(n)
-	s := q.items[i]
-	q.items[i] = q.items[n-1]
-	q.items = q.items[:n-1]
-	q.mu.Unlock()
-	return s, true
-}
-
-func (q *domainQueue) len() int {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-	return len(q.items)
-}

internal/discover/discover_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestCandidatesAreExactLabelsOnly(t *testing.T) {
-	got := candidates("assets.example.com", 200)
+	got := candidates("assets.example.com")
 	if len(got) != 2 || got[0] != "assets" || got[1] != "example" {
 		t.Fatalf("candidates = %v, want [assets example] (no fabricated names)", got)
 	}
@@ -28,29 +28,21 @@ func TestBaseLabelsSkipsNoise(t *testing.T) {
 }
 
 func TestTargets(t *testing.T) {
-	d := New(Config{S3: true, GCS: true, DOSpaces: true, DORegions: []string{"nyc3"}})
+	d := New(Config{})
 	ts := d.targets("example")
-	if len(ts) != 3 {
-		t.Fatalf("got %d targets, want 3", len(ts))
+	if len(ts) != 2 {
+		t.Fatalf("got %d targets, want 2", len(ts))
 	}
-	var s3, gcs, do bool
+	var s3, gcs bool
 	for _, tg := range ts {
 		switch {
 		case strings.Contains(tg.URL, "example.s3.amazonaws.com/?list-type=2"):
 			s3 = true
 		case strings.Contains(tg.URL, "storage.googleapis.com/example"):
 			gcs = true
-		case strings.Contains(tg.URL, "example.nyc3.digitaloceanspaces.com/?list-type=2"):
-			do = true
 		}
 	}
-	if !s3 || !gcs || !do {
-		t.Fatalf("missing targets: s3=%v gcs=%v do=%v (%v)", s3, gcs, do, ts)
-	}
-}
-
-func TestMaxPerDomain(t *testing.T) {
-	if got := candidates("example.com", 5); len(got) > 5 {
-		t.Fatalf("MaxPerDomain not honored: got %d", len(got))
+	if !s3 || !gcs {
+		t.Fatalf("missing targets: s3=%v gcs=%v (%v)", s3, gcs, ts)
 	}
 }

internal/dns/pool.go

@@ -2,7 +2,6 @@ package dns
 
 import (
 	"context"
-	_ "embed"
 	"errors"
 	mathrand "math/rand/v2"
 	"net"
@@ -12,30 +11,7 @@ import (
 	"time"
 )
 
-//go:embed resolvers.txt
-var embeddedResolvers string
-
-// EmbeddedServers returns the compiled-in public resolver list (a snapshot of
-// proabiral/Fresh-Resolvers). Refresh it with:
-//
-//	curl -sL https://raw.githubusercontent.com/proabiral/Fresh-Resolvers/master/resolvers.txt -o internal/dns/resolvers.txt
-//
-// These are untrusted third-party resolvers: they can observe and, in theory,
-// forge DNS answers. Use DNS_SERVERS with your own resolvers for sensitive
-// deployments. A circuit breaker skips ones that stop responding.
-func EmbeddedServers() []string {
-	lines := strings.Split(embeddedResolvers, "\n")
-	out := make([]string, 0, len(lines))
-	for _, l := range lines {
-		if l = strings.TrimSpace(l); l != "" {
-			out = append(out, l)
-		}
-	}
-	return out
-}
-
-// CuratedServers are well-known, reliable public resolvers. They are always
-// included (ahead of the community list) so a mostly-dead list still works.
+// CuratedServers are well-known, reliable public resolvers used by default.
 func CuratedServers() []string {
 	return []string{
 		"1.1.1.1", "1.0.0.1", // Cloudflare
@@ -118,28 +94,13 @@ func NewServerPool(servers []string) *ServerPool {
 // Len reports how many resolvers are in the pool.
 func (p *ServerPool) Len() int { return len(p.servers) }
 
-// Warm probes up to maxProbe randomly chosen resolvers and puts non-responders
-// into cooldown, so a large, partly-stale public list converges on a working
-// subset. It deliberately samples instead of probing the whole list: firing
-// thousands of concurrent DNS queries can overwhelm the host/container
-// resolver and break unrelated lookups. maxProbe <= 0 means all. Returns the
-// healthy count.
-func (p *ServerPool) Warm(ctx context.Context, probeHost string, workers, maxProbe int) int {
+// Warm probes every resolver and puts non-responders into cooldown, so the
+// pool starts on the working subset. Returns the healthy count.
+func (p *ServerPool) Warm(ctx context.Context, probeHost string, workers int) int {
 	if workers <= 0 {
 		workers = 32
 	}
 	servers := p.servers
-	if maxProbe > 0 && maxProbe < len(servers) {
-		order := make([]int, len(servers))
-		for i := range order {
-			order[i] = i
-		}
-		mathrand.Shuffle(len(order), func(i, j int) { order[i], order[j] = order[j], order[i] })
-		servers = make([]*server, maxProbe)
-		for i := range servers {
-			servers[i] = p.servers[order[i]]
-		}
-	}
 	probeTimeout := time.Second
 	if p.attemptTimeout > 0 && p.attemptTimeout < probeTimeout {
 		probeTimeout = p.attemptTimeout

internal/dns/pool_test.go

@@ -30,15 +30,6 @@ func (f *fakeLookup) LookupIP(ctx context.Context, host string) ([]net.IP, error
 	return []net.IP{net.ParseIP("1.2.3.4")}, nil
 }
 
-func TestEmbeddedServers(t *testing.T) {
-	if n := len(EmbeddedServers()); n < 1000 {
-		t.Fatalf("embedded resolver list has %d entries, want thousands", n)
-	}
-	if n := len(CuratedServers()); n < 10 {
-		t.Fatalf("curated resolver list has %d entries, want at least 10", n)
-	}
-}
-
 func TestServerCooldown(t *testing.T) {
 	s := &server{}
 	now := time.Now().UnixNano()
@@ -86,7 +77,7 @@ func TestWarmDisablesDeadResolvers(t *testing.T) {
 	dead := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}}
 	deadServer := &server{r: dead}
 	p := &ServerPool{servers: []*server{deadServer, {r: good}}}
-	healthy := p.Warm(context.Background(), "example.com", 4, 0)
+	healthy := p.Warm(context.Background(), "example.com", 4)
 	if healthy != 1 {
 		t.Fatalf("healthy = %d, want 1", healthy)
 	}

internal/index/index.go

@@ -55,8 +55,6 @@ func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, e
 		switch c.Service {
 		case model.ServiceS3:
 			return x.listS3(ctx, c.Hostname)
-		case model.ServiceOpenSearch:
-			return x.listOpenSearch(ctx, c.Hostname)
 		}
 	case model.ProviderGCP:
 		switch c.Service {
@@ -360,36 +358,6 @@ func (x *Indexer) listFirebase(ctx context.Context, host string) ([]model.Entry,
 	return entries, nil
 }
 
-// --- OpenSearch / Elasticsearch indices ---
-
-func (x *Indexer) listOpenSearch(ctx context.Context, host string) ([]model.Entry, error) {
-	resp, err := x.get(ctx, x.scheme+"://"+host+"/_cat/indices?format=json&h=index,health,status,docs.count,store.size")
-	if err != nil {
-		return nil, err
-	}
-	var rows []struct {
-		Index    string `json:"index"`
-		Health   string `json:"health"`
-		Status   string `json:"status"`
-		Docs     string `json:"docs.count"`
-		StoreSiz string `json:"store.size"`
-	}
-	err = json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&rows)
-	resp.Body.Close()
-	if err != nil {
-		return nil, err
-	}
-	var entries []model.Entry
-	for _, r := range rows {
-		size, _ := strconv.ParseInt(strings.TrimSuffix(r.Docs, " "), 10, 64)
-		entries = append(entries, model.Entry{Kind: "index", Name: r.Index, Size: size, ETag: r.StoreSiz})
-		if len(entries) >= x.maxObjects {
-			break
-		}
-	}
-	return entries, nil
-}
-
 func parseTime(s string) time.Time {
 	if s == "" {
 		return time.Time{}

internal/index/index_test.go

@@ -81,19 +81,6 @@ func TestListFirebase(t *testing.T) {
 	}
 }
 
-func TestListOpenSearch(t *testing.T) {
-	x, host := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_, _ = w.Write([]byte(`[{"index":"logs-2024","docs.count":"100","store.size":"1.2mb"}]`))
-	}))
-	entries, err := x.List(context.Background(), model.Candidate{Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceOpenSearch})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(entries) != 1 || entries[0].Name != "logs-2024" || entries[0].Size != 100 {
-		t.Fatalf("unexpected entries: %+v", entries)
-	}
-}
-
 func TestListBounded(t *testing.T) {
 	var keys strings.Builder
 	for i := 0; i < 100; i++ {

internal/model/model.go

@@ -124,7 +124,6 @@ const (
 	SourceUnknown Source = iota
 	SourceDirectCT
 	SourceCNAME
-	SourceDNSAlias
 	SourceManual
 	SourceDiscovery
 )
@@ -135,8 +134,6 @@ func (s Source) String() string {
 		return "direct_ct"
 	case SourceCNAME:
 		return "cname"
-	case SourceDNSAlias:
-		return "dns_alias"
 	case SourceManual:
 		return "manual"
 	case SourceDiscovery:

internal/pipeline/dnsqueue.go

@@ -1,71 +0,0 @@
-package pipeline
-
-import (
-	mathrand "math/rand/v2"
-	"sync"
-)
-
-// stringSample is a memory-bounded queue that keeps a random sample of recent
-// hostnames instead of a FIFO. A FIFO fills with the first N hostnames seen at
-// startup and drops everything after, so DNS enrichment only ever sees stale
-// hostnames and CNAME discovery stalls. Random replacement makes coverage
-// representative at bounded memory. Not on the certificate hot path.
-type stringSample struct {
-	mu     sync.Mutex
-	items  []string
-	cap    int
-	notify chan struct{}
-	stop   chan struct{}
-	once   sync.Once
-}
-
-func newStringSample(capacity int) *stringSample {
-	return &stringSample{
-		cap:    capacity,
-		notify: make(chan struct{}, 1),
-		stop:   make(chan struct{}),
-	}
-}
-
-func (q *stringSample) push(s string) {
-	q.mu.Lock()
-	if len(q.items) < q.cap {
-		q.items = append(q.items, s)
-	} else {
-		q.items[mathrand.IntN(len(q.items))] = s
-	}
-	q.mu.Unlock()
-	select {
-	case q.notify <- struct{}{}:
-	default:
-	}
-}
-
-func (q *stringSample) pop() (string, bool) {
-	q.mu.Lock()
-	n := len(q.items)
-	if n == 0 {
-		q.mu.Unlock()
-		return "", false
-	}
-	i := mathrand.IntN(n)
-	s := q.items[i]
-	q.items[i] = q.items[n-1]
-	q.items = q.items[:n-1]
-	q.mu.Unlock()
-	return s, true
-}
-
-func (q *stringSample) len() int {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-	return len(q.items)
-}
-
-func (q *stringSample) close() {
-	q.once.Do(func() { close(q.stop) })
-	select {
-	case q.notify <- struct{}{}:
-	default:
-	}
-}

internal/pipeline/pipeline.go

@@ -15,6 +15,7 @@ import (
 	"tea.karlcom.de/karl/bucketeer/internal/dns"
 	"tea.karlcom.de/karl/bucketeer/internal/metrics"
 	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/sample"
 )
 
 // Finding is a fully enriched candidate ready to be persisted.
@@ -32,7 +33,6 @@ type Config struct {
 	DirectQueueSize int
 	DNSQueueSize    int
 	DBQueueSize     int
-	DirectWorkers   int
 	DNSWorkers      int
 	BatchSize       int
 	FlushInterval   time.Duration
@@ -54,7 +54,7 @@ type Pipeline struct {
 	cfg Config
 
 	directQ chan model.Candidate
-	dnsQ    *stringSample
+	dnsQ    *sample.Queue[string]
 	dbQ     chan Finding
 
 	dedup    *dns.TTLCache[bool] // hostnames already queued for DNS
@@ -73,9 +73,6 @@ func New(cfg Config) *Pipeline {
 	if cfg.DBQueueSize <= 0 {
 		cfg.DBQueueSize = 8192
 	}
-	if cfg.DirectWorkers <= 0 {
-		cfg.DirectWorkers = 2
-	}
 	if cfg.DNSWorkers <= 0 {
 		cfg.DNSWorkers = 16
 	}
@@ -94,14 +91,14 @@ func New(cfg Config) *Pipeline {
 	p := &Pipeline{
 		cfg:     cfg,
 		directQ: make(chan model.Candidate, cfg.DirectQueueSize),
-		dnsQ:    newStringSample(cfg.DNSQueueSize),
+		dnsQ:    sample.New[string](cfg.DNSQueueSize),
 		dbQ:     make(chan Finding, cfg.DBQueueSize),
 		dedup:   dns.NewTTLCache[bool](cfg.DedupSize, 64),
 		seen:    dns.NewTTLCache[bool](cfg.DedupSize, 64),
 	}
 	if p.cfg.Metrics != nil {
 		p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) })
-		p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(p.dnsQ.len()) })
+		p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(p.dnsQ.Len()) })
 		p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) })
 	}
 	return p
@@ -109,7 +106,7 @@ func New(cfg Config) *Pipeline {
 
 // Start launches the worker pools. Close must be called to drain.
 func (p *Pipeline) Start(ctx context.Context) {
-	for i := 0; i < p.cfg.DirectWorkers; i++ {
+	for i := 0; i < 2; i++ {
 		p.producer.Add(1)
 		go p.directWorker(ctx)
 	}
@@ -123,14 +120,14 @@ func (p *Pipeline) Start(ctx context.Context) {
 
 // Depths reports the current length of the direct, DNS and DB queues.
 func (p *Pipeline) Depths() (direct, dns, db int) {
-	return len(p.directQ), p.dnsQ.len(), len(p.dbQ)
+	return len(p.directQ), p.dnsQ.Len(), len(p.dbQ)
 }
 
 // Close stops intake and drains all queues. It must only be called after the
 // CertSpotter monitor has stopped calling HandleIdentifiers.
 func (p *Pipeline) Close() {
 	close(p.directQ)
-	p.dnsQ.close()
+	p.dnsQ.Close()
 	p.producer.Wait()
 	close(p.dbQ)
 	p.workers.Wait()
@@ -179,7 +176,7 @@ func (p *Pipeline) HandleHost(host string) {
 		return
 	}
 	p.dedup.Set(host, true, p.cfg.DedupTTL)
-	p.dnsQ.push(host)
+	p.dnsQ.Push(host)
 	if p.cfg.Discover != nil {
 		p.cfg.Discover.Submit(host)
 	}
@@ -242,12 +239,12 @@ func (p *Pipeline) dnsWorker(ctx context.Context) {
 		select {
 		case <-ctx.Done():
 			return
-		case <-p.dnsQ.stop:
+		case <-p.dnsQ.Stop():
 			return
-		case <-p.dnsQ.notify:
+		case <-p.dnsQ.Notify():
 		}
 		for {
-			host, ok := p.dnsQ.pop()
+			host, ok := p.dnsQ.Pop()
 			if !ok {
 				break
 			}

internal/pipeline/pipeline_test.go

@@ -269,7 +269,7 @@ func TestDNSDedup(t *testing.T) {
 	p := New(Config{DNSQueueSize: 8, Metrics: m})
 	p.HandleHost("files.example.com")
 	p.HandleHost("files.example.com")
-	if got := p.dnsQ.len(); got != 1 {
+	if got := p.dnsQ.Len(); got != 1 {
 		t.Fatalf("dns queue len = %d, want 1", got)
 	}
 }

internal/probe/probe.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	mathrand "math/rand/v2"
 	"net"
 	"net/http"
 	"net/url"
@@ -32,6 +31,7 @@ import (
 	"tea.karlcom.de/karl/bucketeer/internal/index"
 	"tea.karlcom.de/karl/bucketeer/internal/metrics"
 	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/sample"
 )
 
 // Result is the outcome of probing one candidate.
@@ -75,61 +75,12 @@ type Config struct {
 
 type Prober struct {
 	cfg    Config
-	sample *sampleQueue
+	sample *sample.Queue[model.Candidate]
 	client *http.Client
 	stop   chan struct{}
 	wg     sync.WaitGroup
 }
 
-// sampleQueue is a memory-bounded queue that keeps a random sample of recent
-// submissions rather than a FIFO, so one high-volume hostname template cannot
-// starve every other asset type. It is not on the certificate hot path.
-type sampleQueue struct {
-	mu     sync.Mutex
-	items  []model.Candidate
-	cap    int
-	notify chan struct{}
-}
-
-func newSampleQueue(capacity int) *sampleQueue {
-	return &sampleQueue{cap: capacity, notify: make(chan struct{}, 1)}
-}
-
-func (q *sampleQueue) push(c model.Candidate) {
-	q.mu.Lock()
-	if len(q.items) < q.cap {
-		q.items = append(q.items, c)
-	} else {
-		q.items[mathrand.IntN(len(q.items))] = c
-	}
-	q.mu.Unlock()
-	select {
-	case q.notify <- struct{}{}:
-	default:
-	}
-}
-
-func (q *sampleQueue) len() int {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-	return len(q.items)
-}
-
-func (q *sampleQueue) pop() (model.Candidate, bool) {
-	q.mu.Lock()
-	n := len(q.items)
-	if n == 0 {
-		q.mu.Unlock()
-		return model.Candidate{}, false
-	}
-	i := mathrand.IntN(n)
-	c := q.items[i]
-	q.items[i] = q.items[n-1]
-	q.items = q.items[:n-1]
-	q.mu.Unlock()
-	return c, true
-}
-
 func New(cfg Config) *Prober {
 	if cfg.Workers <= 0 {
 		cfg.Workers = 4
@@ -142,7 +93,7 @@ func New(cfg Config) *Prober {
 	}
 	return &Prober{
 		cfg:    cfg,
-		sample: newSampleQueue(cfg.Queue),
+		sample: sample.New[model.Candidate](cfg.Queue),
 		stop:   make(chan struct{}),
 		client: &http.Client{
 			Timeout: cfg.Timeout,
@@ -178,7 +129,7 @@ func (p *Prober) Submit(c model.Candidate) {
 	if p.cfg.Metrics != nil {
 		p.cfg.Metrics.ProbeAttempts.Add(1)
 	}
-	p.sample.push(c)
+	p.sample.Push(c)
 }
 
 // Close stops the workers after any in-flight probe. It must be called before
@@ -188,7 +139,7 @@ func (p *Prober) Close() { close(p.stop) }
 func (p *Prober) Wait() { p.wg.Wait() }
 
 // QueueLen reports the current probe sample depth.
-func (p *Prober) QueueLen() int { return p.sample.len() }
+func (p *Prober) QueueLen() int { return p.sample.Len() }
 
 func (p *Prober) worker(ctx context.Context) {
 	defer p.wg.Done()
@@ -198,10 +149,10 @@ func (p *Prober) worker(ctx context.Context) {
 			return
 		case <-p.stop:
 			return
-		case <-p.sample.notify:
+		case <-p.sample.Notify():
 		}
 		for {
-			c, ok := p.sample.pop()
+			c, ok := p.sample.Pop()
 			if !ok {
 				break
 			}
@@ -225,12 +176,6 @@ func (p *Prober) worker(ctx context.Context) {
 	}
 }
 
-// Probe is the (exposure, detail) convenience view.
-func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
-	r := p.ProbeDetailed(ctx, c)
-	return r.Exposure, r.Detail
-}
-
 // ProbeDetailed runs the vendor-appropriate check and reports capabilities.
 func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result {
 	// Deep, read-only protocol checks take precedence for database services.
@@ -469,8 +414,3 @@ func classifyError(err error) (model.ExposureState, string) {
 	}
 	return model.ExposureUnknown, err.Error()
 }
-
-// Describe is a helper for logs/tests.
-func Describe(s model.ExposureState) string {
-	return fmt.Sprintf("exposure=%s", s)
-}

internal/probe/probe_test.go

@@ -38,11 +38,11 @@ func TestProbeStorageStatuses(t *testing.T) {
 	for _, tc := range cases {
 		p := New(Config{Workers: 1, Timeout: time.Second})
 		host := serverHost(t, statusHandler(tc.status))
-		got, detail := p.Probe(context.Background(), model.Candidate{
+		r := p.ProbeDetailed(context.Background(), model.Candidate{
 			Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
 		})
-		if got != tc.want {
-			t.Errorf("status %d: got %s (%s), want %s", tc.status, got, detail, tc.want)
+		if r.Exposure != tc.want {
+			t.Errorf("status %d: got %s (%s), want %s", tc.status, r.Exposure, r.Detail, tc.want)
 		}
 	}
 }
@@ -54,29 +54,29 @@ func TestProbeS3RegionRedirect(t *testing.T) {
 	})
 	p := New(Config{Workers: 1, Timeout: time.Second})
 	host := serverHost(t, h)
-	got, detail := p.Probe(context.Background(), model.Candidate{
+	r := p.ProbeDetailed(context.Background(), model.Candidate{
 		Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
 	})
-	if got != model.ExposureReachable || !strings.Contains(detail, "eu-central-1") {
-		t.Fatalf("got %s (%s), want reachable with region", got, detail)
+	if r.Exposure != model.ExposureReachable || !strings.Contains(r.Detail, "eu-central-1") {
+		t.Fatalf("got %s (%s), want reachable with region", r.Exposure, r.Detail)
 	}
 }
 
 func TestProbeFirebase(t *testing.T) {
 	p := New(Config{Workers: 1, Timeout: time.Second})
 	host := serverHost(t, statusHandler(http.StatusUnauthorized))
-	got, _ := p.Probe(context.Background(), model.Candidate{
+	r := p.ProbeDetailed(context.Background(), model.Candidate{
 		Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
 	})
-	if got != model.ExposureAuthRequired {
-		t.Fatalf("got %s, want auth_required", got)
+	if r.Exposure != model.ExposureAuthRequired {
+		t.Fatalf("got %s, want auth_required", r.Exposure)
 	}
 	host2 := serverHost(t, statusHandler(http.StatusOK))
-	got, _ = p.Probe(context.Background(), model.Candidate{
+	r = p.ProbeDetailed(context.Background(), model.Candidate{
 		Hostname: host2, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
 	})
-	if got != model.ExposurePublicRead {
-		t.Fatalf("got %s, want public_read", got)
+	if r.Exposure != model.ExposurePublicRead {
+		t.Fatalf("got %s, want public_read", r.Exposure)
 	}
 }
 
@@ -90,11 +90,11 @@ func TestProbeAzureContainerDiscovery(t *testing.T) {
 	})
 	p := New(Config{Workers: 1, Timeout: time.Second})
 	host := serverHost(t, h)
-	state, detail := p.Probe(context.Background(), model.Candidate{
+	r := p.ProbeDetailed(context.Background(), model.Candidate{
 		Hostname: host, Resource: "acct", Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
 	})
-	if state != model.ExposurePublicRead || !strings.Contains(detail, "images") {
-		t.Fatalf("got %s (%s), want public_read via container 'images'", state, detail)
+	if r.Exposure != model.ExposurePublicRead || !strings.Contains(r.Detail, "images") {
+		t.Fatalf("got %s (%s), want public_read via container 'images'", r.Exposure, r.Detail)
 	}
 }
 

internal/sample/queue.go

@@ -0,0 +1,77 @@
+// Package sample provides a bounded queue that keeps a random sample of recent
+// items instead of a FIFO. A FIFO fills with the first N items and drops the
+// rest, which starves later work; random replacement gives representative
+// coverage at bounded memory. Not for hot paths.
+package sample
+
+import (
+	mathrand "math/rand/v2"
+	"sync"
+)
+
+type Queue[T any] struct {
+	mu     sync.Mutex
+	items  []T
+	cap    int
+	notify chan struct{}
+	stop   chan struct{}
+	once   sync.Once
+}
+
+func New[T any](capacity int) *Queue[T] {
+	if capacity < 1 {
+		capacity = 1
+	}
+	return &Queue[T]{
+		cap:    capacity,
+		notify: make(chan struct{}, 1),
+		stop:   make(chan struct{}),
+	}
+}
+
+func (q *Queue[T]) Push(v T) {
+	q.mu.Lock()
+	if len(q.items) < q.cap {
+		q.items = append(q.items, v)
+	} else {
+		q.items[mathrand.IntN(len(q.items))] = v
+	}
+	q.mu.Unlock()
+	select {
+	case q.notify <- struct{}{}:
+	default:
+	}
+}
+
+func (q *Queue[T]) Pop() (T, bool) {
+	q.mu.Lock()
+	n := len(q.items)
+	if n == 0 {
+		q.mu.Unlock()
+		var zero T
+		return zero, false
+	}
+	i := mathrand.IntN(n)
+	v := q.items[i]
+	q.items[i] = q.items[n-1]
+	q.items = q.items[:n-1]
+	q.mu.Unlock()
+	return v, true
+}
+
+func (q *Queue[T]) Len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return len(q.items)
+}
+
+func (q *Queue[T]) Notify() <-chan struct{} { return q.notify }
+func (q *Queue[T]) Stop() <-chan struct{}   { return q.stop }
+
+func (q *Queue[T]) Close() {
+	q.once.Do(func() { close(q.stop) })
+	select {
+	case q.notify <- struct{}{}:
+	default:
+	}
+}

internal/store/store.go

@@ -47,8 +47,6 @@ func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
 
 func (s *Store) Close() { s.pool.Close() }
 
-func (s *Store) Pool() *pgxpool.Pool { return s.pool }
-
 // PoolStats reports pgx pool utilization for logging/metrics.
 func (s *Store) PoolStats() (total, idle, acquired int32) {
 	st := s.pool.Stat()