bucket discovery engine: CT domain -> candidate names -> listing index

CMS SANs are custom domains, not bucket names, so matching SANs finds almost
no buckets. This adds internal/discover: from each non-matching hostname it
derives candidate names (domain label + subdomain label + ~40 suffixes) plus a
built-in wordlist, and anonymously tests listing on S3, GCS and (optionally)
DigitalOcean Spaces. Every 200 is indexed (keys/sizes/dates only).

Bounded hard: global BUCKET_SCAN_QPS rate limit, 12h per-name dedup, random
sampled domain queue, MaxPerDomain cap. Never downloads contents.

Fix: GCS candidates now use the virtual-host form
(<bucket>.storage.googleapis.com) so buckets no longer collide on the
cloud_assets unique key; previously they all shared storage.googleapis.com and
overwrote each other.

Live: 30 open buckets in ~5 min (e.g. estimator-dev, image-prod,
update-media, mos-images S3; several GCS buckets), 11k+ objects indexed.

internal/config/config.go

@@ -39,6 +39,12 @@ type Config struct {
 	IndexEnabled    bool
 	IndexMaxObjects int
 
+	BucketScanEnabled bool
+	BucketScanQPS     int
+	BucketScanWorkers int
+	BucketScanQueue   int
+	BucketScanDO      bool
+
 	MetricsAddr   string
 	UIUser        string
 	UIPass        string
@@ -57,38 +63,43 @@ const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.js
 
 func Load() (*Config, error) {
 	c := &Config{
-		DatabaseURL:      os.Getenv("DATABASE_URL"),
-		DBMaxConns:       envInt("DB_MAX_CONNS", 8),
-		DNSServers:       envList("DNS_SERVERS", nil),
-		DNSWorkers:       envInt("DNS_WORKERS", 64),
-		DNSTimeout:       envDuration("DNS_TIMEOUT", 5*time.Second),
-		DNSCacheSize:     envInt("DNS_CACHE_SIZE", 1<<18),
-		DedupSize:        envInt("DEDUP_SIZE", 1<<18),
-		DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
-		DNSPositiveTTL:   envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
-		DNSNegativeTTL:   envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
-		DirectQueueSize:  envInt("DIRECT_QUEUE_SIZE", 8192),
-		DNSQueueSize:     envInt("DNS_QUEUE_SIZE", 262144),
-		DBQueueSize:      envInt("DB_QUEUE_SIZE", 8192),
-		DBBatchSize:      envInt("DB_BATCH_SIZE", 500),
-		DBFlushInterval:  envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
-		ActiveProbing:    envBool("ACTIVE_PROBING", false),
-		ProbeWorkers:     envInt("PROBE_WORKERS", 8),
-		ProbeTimeout:     envDuration("PROBE_TIMEOUT", 5*time.Second),
-		ProbeQueueSize:   envInt("PROBE_QUEUE_SIZE", 1<<16),
-		ProbeDeep:        envBool("PROBE_DEEP", true),
-		IndexEnabled:     envBool("INDEX_ENABLED", true),
-		IndexMaxObjects:  envInt("INDEX_MAX_OBJECTS", 1000),
-		MetricsAddr:      envString("METRICS_ADDR", ":9090"),
-		UIUser:           envString("UI_USER", ""),
-		UIPass:           envString("UI_PASS", ""),
-		LogLevel:         envString("LOG_LEVEL", "info"),
-		LogFormat:        envString("LOG_FORMAT", "text"),
-		StatsInterval:    envDuration("STATS_INTERVAL", 60*time.Second),
-		LogListSource:    envString("LOG_LIST_SOURCE", DefaultLogListSource),
-		StartAtEnd:       envBool("START_AT_END", false),
-		ShutdownTimeout:  envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
-		DedupeTTL:        envDuration("DEDUP_TTL", time.Hour),
+		DatabaseURL:       os.Getenv("DATABASE_URL"),
+		DBMaxConns:        envInt("DB_MAX_CONNS", 8),
+		DNSServers:        envList("DNS_SERVERS", nil),
+		DNSWorkers:        envInt("DNS_WORKERS", 64),
+		DNSTimeout:        envDuration("DNS_TIMEOUT", 5*time.Second),
+		DNSCacheSize:      envInt("DNS_CACHE_SIZE", 1<<18),
+		DedupSize:         envInt("DEDUP_SIZE", 1<<18),
+		DNSMaxCNAMEdepth:  envInt("DNS_MAX_CNAME_DEPTH", 8),
+		DNSPositiveTTL:    envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
+		DNSNegativeTTL:    envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
+		DirectQueueSize:   envInt("DIRECT_QUEUE_SIZE", 8192),
+		DNSQueueSize:      envInt("DNS_QUEUE_SIZE", 262144),
+		DBQueueSize:       envInt("DB_QUEUE_SIZE", 8192),
+		DBBatchSize:       envInt("DB_BATCH_SIZE", 500),
+		DBFlushInterval:   envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
+		ActiveProbing:     envBool("ACTIVE_PROBING", false),
+		ProbeWorkers:      envInt("PROBE_WORKERS", 8),
+		ProbeTimeout:      envDuration("PROBE_TIMEOUT", 5*time.Second),
+		ProbeQueueSize:    envInt("PROBE_QUEUE_SIZE", 1<<16),
+		ProbeDeep:         envBool("PROBE_DEEP", true),
+		IndexEnabled:      envBool("INDEX_ENABLED", true),
+		IndexMaxObjects:   envInt("INDEX_MAX_OBJECTS", 1000),
+		BucketScanEnabled: envBool("BUCKET_SCAN_ENABLED", true),
+		BucketScanQPS:     envInt("BUCKET_SCAN_QPS", 50),
+		BucketScanWorkers: envInt("BUCKET_SCAN_WORKERS", 16),
+		BucketScanQueue:   envInt("BUCKET_SCAN_QUEUE", 65536),
+		BucketScanDO:      envBool("BUCKET_SCAN_DO", false),
+		MetricsAddr:       envString("METRICS_ADDR", ":9090"),
+		UIUser:            envString("UI_USER", ""),
+		UIPass:            envString("UI_PASS", ""),
+		LogLevel:          envString("LOG_LEVEL", "info"),
+		LogFormat:         envString("LOG_FORMAT", "text"),
+		StatsInterval:     envDuration("STATS_INTERVAL", 60*time.Second),
+		LogListSource:     envString("LOG_LIST_SOURCE", DefaultLogListSource),
+		StartAtEnd:        envBool("START_AT_END", false),
+		ShutdownTimeout:   envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		DedupeTTL:         envDuration("DEDUP_TTL", time.Hour),
 	}
 	if err := c.validate(); err != nil {
 		return nil, err

internal/discover/discover.go

@@ -0,0 +1,415 @@
+// Package discover turns the domains seen in CT into candidate object-storage
+// bucket names and tests them anonymously. This is the engine that actually
+// finds open buckets: certificate SANs are custom domains, not bucket names, so
+// we generate plausible names and ask the providers' list APIs.
+//
+// It is aggressively bounded: a global rate limiter caps outbound requests,
+// candidate names are deduplicated with a TTL cache, and the domain intake is a
+// random sample so no single domain floods the queue. Contents are never
+// downloaded; only the list API is called.
+package discover
+
+import (
+	"context"
+	"io"
+	"math/rand/v2"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/index"
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+type Config struct {
+	QPS          int // global outbound request rate
+	Workers      int // concurrent candidate probes
+	Queue        int // sampled domain backlog
+	S3           bool
+	GCS          bool
+	DOSpaces     bool
+	DORegions    []string
+	MaxPerDomain int // cap candidates generated per domain
+
+	Indexer *index.Indexer
+	Metrics *metrics.Metrics
+	// OnOpen is called for every anonymously listable bucket with its metadata
+	// index. contents are never downloaded.
+	OnOpen func(ctx context.Context, c model.Candidate, detail string, entries []model.Entry)
+}
+
+type Discoverer struct {
+	cfg    Config
+	client *http.Client
+	q      *domainQueue
+	names  *domainQueue
+	tokens chan struct{}
+	seen   *dns.TTLCache[bool] // candidate names already tested
+	stop   chan struct{}
+	wg     sync.WaitGroup
+}
+
+func New(cfg Config) *Discoverer {
+	if cfg.QPS <= 0 {
+		cfg.QPS = 50
+	}
+	if cfg.Workers <= 0 {
+		cfg.Workers = 16
+	}
+	if cfg.Queue <= 0 {
+		cfg.Queue = 65536
+	}
+	if cfg.MaxPerDomain <= 0 {
+		cfg.MaxPerDomain = 48
+	}
+	return &Discoverer{
+		cfg:    cfg,
+		client: &http.Client{Timeout: 6 * time.Second},
+		q:      newDomainQueue(cfg.Queue),
+		names:  newDomainQueue(cfg.Queue),
+		tokens: make(chan struct{}, max(cfg.QPS, 1)),
+		seen:   dns.NewTTLCache[bool](1<<20, 64),
+		stop:   make(chan struct{}),
+	}
+}
+
+// Submit offers a hostname to the discovery engine. Duplicate registrable
+// domains and already-tested candidate names are dropped cheaply.
+func (d *Discoverer) Submit(host string) {
+	d.q.push(host)
+}
+
+func (d *Discoverer) Start(ctx context.Context) {
+	go d.refill(ctx)
+	for i := 0; i < d.cfg.Workers; i++ {
+		d.wg.Add(1)
+		go d.worker(ctx)
+	}
+}
+
+func (d *Discoverer) Close() { close(d.stop) }
+func (d *Discoverer) Wait()  { d.wg.Wait() }
+
+// QueueLen reports the sampled domain backlog.
+func (d *Discoverer) QueueLen() int { return d.q.len() }
+
+func (d *Discoverer) refill(ctx context.Context) {
+	period := time.Second / time.Duration(d.cfg.QPS)
+	if period <= 0 {
+		period = time.Millisecond
+	}
+	t := time.NewTicker(period)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-d.stop:
+			return
+		case <-t.C:
+			select {
+			case d.tokens <- struct{}{}:
+			default:
+			}
+		}
+	}
+}
+
+func (d *Discoverer) acquire(ctx context.Context) bool {
+	select {
+	case <-ctx.Done():
+		return false
+	case <-d.stop:
+		return false
+	case <-d.tokens:
+		return true
+	}
+}
+
+func (d *Discoverer) worker(ctx context.Context) {
+	defer d.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-d.stop:
+			return
+		case <-d.q.notify:
+		case <-d.names.notify:
+		}
+		for {
+			host, ok := d.q.pop()
+			if !ok {
+				break
+			}
+			d.scanDomain(ctx, host)
+		}
+		for {
+			name, ok := d.names.pop()
+			if !ok {
+				break
+			}
+			d.testName(ctx, name)
+		}
+	}
+}
+
+func (d *Discoverer) scanDomain(ctx context.Context, host string) {
+	for _, name := range candidates(host, d.cfg.MaxPerDomain) {
+		if ctx.Err() != nil {
+			return
+		}
+		d.testName(ctx, name)
+	}
+}
+
+// SubmitName tests a specific candidate bucket name (e.g. from a wordlist).
+func (d *Discoverer) SubmitName(name string) {
+	if validBucketName(name) {
+		d.names.push(name)
+	}
+}
+
+func (d *Discoverer) testName(ctx context.Context, name string) {
+	if _, dup := d.seen.Get(name); dup {
+		return
+	}
+	d.seen.Set(name, true, 12*time.Hour)
+	for _, t := range d.targets(name) {
+		if d.cfg.Metrics != nil {
+			d.cfg.Metrics.BucketCandidates.Add(1)
+		}
+		if !d.acquire(ctx) {
+			return
+		}
+		d.test(ctx, t)
+	}
+}
+
+// Target is one provider listing endpoint for a candidate name.
+type Target struct {
+	Name     string
+	Provider model.Provider
+	Service  model.Service
+	Hostname string // hostname for the indexer/probe candidate
+	URL      string // anonymous list URL
+	Region   string
+}
+
+func (d *Discoverer) targets(name string) []Target {
+	var out []Target
+	if d.cfg.S3 {
+		out = append(out, Target{
+			Name: name, Provider: model.ProviderAWS, Service: model.ServiceS3,
+			Hostname: name + ".s3.amazonaws.com",
+			URL:      "https://" + name + ".s3.amazonaws.com/?list-type=2&max-keys=1",
+		})
+	}
+	if d.cfg.GCS {
+		// Use the virtual-host form so each bucket has a unique hostname (the
+		// path form makes every bucket share storage.googleapis.com and collide
+		// on the cloud_assets unique key).
+		out = append(out, Target{
+			Name: name, Provider: model.ProviderGCP, Service: model.ServiceObjectStorage,
+			Hostname: name + ".storage.googleapis.com",
+			URL:      "https://storage.googleapis.com/" + name + "?max-keys=1",
+		})
+	}
+	if d.cfg.DOSpaces {
+		for _, region := range d.cfg.DORegions {
+			host := name + "." + region + ".digitaloceanspaces.com"
+			out = append(out, Target{
+				Name: name, Provider: model.ProviderDigitalOcean, Service: model.ServiceObjectStorage,
+				Hostname: host, Region: region,
+				URL: "https://" + host + "/?list-type=2&max-keys=1",
+			})
+		}
+	}
+	return out
+}
+
+func (d *Discoverer) test(ctx context.Context, t Target) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, t.URL, nil)
+	if err != nil {
+		return
+	}
+	resp, err := d.client.Do(req)
+	if err != nil {
+		return
+	}
+	_, _ = io.CopyN(io.Discard, resp.Body, 512)
+	finalHost := resp.Request.URL.Host
+	resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return
+	}
+	// Listable. Use the post-redirect host (S3 global redirects to a region).
+	c := model.Candidate{
+		Hostname: t.Hostname, Original: t.Hostname, Resource: t.Name, Region: t.Region,
+		Provider: t.Provider, Service: t.Service, ResourceSpecific: true, Source: model.SourceDiscovery,
+	}
+	if finalHost != "" && strings.HasSuffix(finalHost, ".amazonaws.com") {
+		c.Hostname = finalHost
+	}
+	detail := "discovered open bucket via name enumeration (" + t.Hostname + ")"
+	if d.cfg.Metrics != nil {
+		d.cfg.Metrics.BucketOpen.Add(1)
+	}
+	if d.cfg.OnOpen == nil {
+		return
+	}
+	var entries []model.Entry
+	if d.cfg.Indexer != nil {
+		if e, err := d.cfg.Indexer.List(ctx, c); err == nil {
+			entries = e
+		}
+	}
+	d.cfg.OnOpen(ctx, c, detail, entries)
+}
+
+// --- candidate generation ---
+
+var suffixes = []string{
+	"", "-assets", "-static", "-backups", "-backup", "-images", "-img", "-media",
+	"-prod", "-dev", "-staging", "-test", "-files", "-uploads", "-public", "-cdn",
+	"-data", "-content", "-site", "-web", "-www", "-app", "-store", "-downloads",
+	"-photos", "-videos", "-docs", "-documents", "-archive", "-logs", "-artifacts",
+	"-build", "-deploy", "-release", "-db", "-database", "-sql", "-exports", "-reports",
+	"-assets-prod", "-assets-dev", "-static-prod", "-uploads-prod",
+}
+
+// Wordlist is a bounded set of common global bucket names tested once each.
+func Wordlist() []string {
+	return []string{
+		"backups", "backup", "images", "image", "img", "media", "static", "assets",
+		"uploads", "files", "file", "data", "content", "public", "cdn", "www", "web",
+		"website", "app", "store", "downloads", "docs", "documents", "archive",
+		"logs", "artifacts", "build", "deploy", "release", "db", "database", "sql",
+		"dump", "exports", "reports", "photos", "videos", "dev", "test", "staging",
+		"prod", "production", "internal", "private", "secure", "userdata", "usercontent",
+		"thumbnails", "avatars", "profiles", "attachments", "tmp", "temp", "cache",
+		"config", "secrets", "keys", "certificates", "certs", "gifs", "png", "jpeg",
+		"download", "uploads-test", "uploads-dev", "assets-test", "assets-dev",
+		"static-assets", "web-assets", "app-assets", "site-assets", "media-assets",
+	}
+}
+
+// baseLabels returns plausible bucket-name stems from a hostname: the leftmost
+// label and the registrable-domain label.
+func baseLabels(host string) []string {
+	host = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(host), "."))
+	labels := strings.Split(host, ".")
+	if len(labels) < 2 {
+		return nil
+	}
+	cands := []string{labels[0], labels[len(labels)-2]}
+	seen := map[string]bool{}
+	out := cands[:0]
+	for _, c := range cands {
+		if validLabel(c) && !seen[c] && !skipLabel[c] {
+			seen[c] = true
+			out = append(out, c)
+		}
+	}
+	return out
+}
+
+// Only genuinely non-descriptive labels are skipped; service-ish subdomains
+// (assets, cdn, media, ...) are kept because they are common bucket names.
+var skipLabel = map[string]bool{
+	"www": true, "api": true, "mail": true, "email": true, "smtp": true,
+	"ns": true, "mx": true, "localhost": true,
+}
+
+func candidates(host string, max int) []string {
+	var out []string
+	for _, b := range baseLabels(host) {
+		for _, s := range suffixes {
+			name := b + s
+			if validBucketName(name) {
+				out = append(out, name)
+			}
+			if max > 0 && len(out) >= max {
+				return out
+			}
+		}
+	}
+	return out
+}
+
+func validLabel(s string) bool {
+	if len(s) < 3 || len(s) > 40 {
+		return false
+	}
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {
+			return false
+		}
+	}
+	return s[0] != '-' && s[len(s)-1] != '-'
+}
+
+func validBucketName(s string) bool {
+	if len(s) < 3 || len(s) > 63 {
+		return false
+	}
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '.') {
+			return false
+		}
+	}
+	return s[0] != '-' && s[0] != '.' && s[len(s)-1] != '-' && s[len(s)-1] != '.'
+}
+
+// --- sampled domain queue ---
+
+type domainQueue struct {
+	mu     sync.Mutex
+	items  []string
+	cap    int
+	notify chan struct{}
+}
+
+func newDomainQueue(capacity int) *domainQueue {
+	return &domainQueue{cap: capacity, notify: make(chan struct{}, 1)}
+}
+
+func (q *domainQueue) push(s string) {
+	q.mu.Lock()
+	if len(q.items) < q.cap {
+		q.items = append(q.items, s)
+	} else {
+		q.items[rand.IntN(len(q.items))] = s
+	}
+	q.mu.Unlock()
+	select {
+	case q.notify <- struct{}{}:
+	default:
+	}
+}
+
+func (q *domainQueue) pop() (string, bool) {
+	q.mu.Lock()
+	n := len(q.items)
+	if n == 0 {
+		q.mu.Unlock()
+		return "", false
+	}
+	i := rand.IntN(n)
+	s := q.items[i]
+	q.items[i] = q.items[n-1]
+	q.items = q.items[:n-1]
+	q.mu.Unlock()
+	return s, true
+}
+
+func (q *domainQueue) len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return len(q.items)
+}

internal/discover/discover_test.go

@@ -0,0 +1,63 @@
+package discover
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestCandidates(t *testing.T) {
+	got := candidates("assets.example.com", 200)
+	set := map[string]bool{}
+	for _, c := range got {
+		set[c] = true
+	}
+	for _, want := range []string{"example", "example-assets", "example-backups", "example-static"} {
+		if !set[want] {
+			t.Fatalf("candidates missing %q (got %v)", want, got)
+		}
+	}
+	for _, c := range got {
+		if !validBucketName(c) {
+			t.Fatalf("invalid candidate %q", c)
+		}
+	}
+}
+
+func TestBaseLabelsSkipsNoise(t *testing.T) {
+	// www is skipped, so only the registrable label remains.
+	got := baseLabels("www.example.com")
+	if len(got) != 1 || got[0] != "example" {
+		t.Fatalf("baseLabels = %v, want [example]", got)
+	}
+	if baseLabels("example") != nil {
+		t.Fatal("single label must yield no base")
+	}
+}
+
+func TestTargets(t *testing.T) {
+	d := New(Config{S3: true, GCS: true, DOSpaces: true, DORegions: []string{"nyc3"}})
+	ts := d.targets("example")
+	if len(ts) != 3 {
+		t.Fatalf("got %d targets, want 3", len(ts))
+	}
+	var s3, gcs, do bool
+	for _, tg := range ts {
+		switch {
+		case strings.Contains(tg.URL, "example.s3.amazonaws.com/?list-type=2"):
+			s3 = true
+		case strings.Contains(tg.URL, "storage.googleapis.com/example"):
+			gcs = true
+		case strings.Contains(tg.URL, "example.nyc3.digitaloceanspaces.com/?list-type=2"):
+			do = true
+		}
+	}
+	if !s3 || !gcs || !do {
+		t.Fatalf("missing targets: s3=%v gcs=%v do=%v (%v)", s3, gcs, do, ts)
+	}
+}
+
+func TestMaxPerDomain(t *testing.T) {
+	if got := candidates("example.com", 5); len(got) != 5 {
+		t.Fatalf("MaxPerDomain not honored: got %d", len(got))
+	}
+}

internal/metrics/metrics.go

@@ -41,6 +41,9 @@ type Metrics struct {
 	ProbePublic    atomic.Uint64
 	EntriesIndexed atomic.Uint64
 
+	BucketCandidates atomic.Uint64
+	BucketOpen       atomic.Uint64
+
 	gauges []gauge
 }
 
@@ -86,6 +89,8 @@ func (m *Metrics) Handler() http.Handler {
 		counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
 		counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
 		counter("entries_indexed_total", "Metadata entries indexed from readable resources", m.EntriesIndexed.Load())
+		counter("bucket_candidates_total", "Candidate bucket names tested", m.BucketCandidates.Load())
+		counter("bucket_open_total", "Open (listable) buckets discovered by name enumeration", m.BucketOpen.Load())
 		for _, g := range m.gauges {
 			fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
 		}

internal/model/model.go

@@ -126,6 +126,7 @@ const (
 	SourceCNAME
 	SourceDNSAlias
 	SourceManual
+	SourceDiscovery
 )
 
 func (s Source) String() string {
@@ -138,6 +139,8 @@ func (s Source) String() string {
 		return "dns_alias"
 	case SourceManual:
 		return "manual"
+	case SourceDiscovery:
+		return "discovery"
 	default:
 		return "unknown"
 	}

internal/pipeline/pipeline.go

@@ -42,6 +42,10 @@ type Config struct {
 	Enricher *dns.Enricher
 	Write    WriteFunc
 	Metrics  *metrics.Metrics
+
+	// Discover, when set, receives non-matching hostnames for bucket-name
+	// enumeration (see internal/discover).
+	Discover interface{ Submit(host string) }
 }
 
 // Pipeline is the certificate-to-database data flow. It is safe to call
@@ -176,6 +180,9 @@ func (p *Pipeline) HandleHost(host string) {
 	}
 	p.dedup.Set(host, true, p.cfg.DedupTTL)
 	p.dnsQ.push(host)
+	if p.cfg.Discover != nil {
+		p.cfg.Discover.Submit(host)
+	}
 }
 
 func (p *Pipeline) drop() {