ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

internal/classify/aws.go

@@ -0,0 +1,148 @@
+package classify
+
+import (
+	"strings"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match {
+	return model.Match{
+		Provider:         p,
+		Service:          s,
+		PublicSemantic:   sem,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+func classifyAWS(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {
+		return model.Match{}, false
+	}
+	if m, ok := classifyAWSS3(s); ok {
+		return m, true
+	}
+
+	// Service-after-region forms: <resource>.<region>.<service>.amazonaws.com
+	if n >= 5 {
+		switch s.label(n - 3) {
+		case "rds":
+			return awsResourceAfterRegion(s, model.ServiceRDS)
+		case "docdb":
+			return awsResourceAfterRegion(s, model.ServiceDocumentDB)
+		case "neptune":
+			return awsResourceAfterRegion(s, model.ServiceNeptune)
+		case "redshift", "redshift-serverless":
+			return awsResourceAfterRegion(s, model.ServiceRedshift)
+		case "es":
+			first := s.label(0)
+			switch {
+			case strings.HasPrefix(first, "search-"):
+				return aws(model.ProviderAWS, model.ServiceOpenSearch,
+					model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true
+			case strings.HasPrefix(first, "vpc-"):
+				return aws(model.ProviderAWS, model.ServiceOpenSearch,
+					model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true
+			}
+		case "cache":
+			return aws(model.ProviderAWS, model.ServiceElastiCache,
+				model.PublicSemanticNone, s.label(0), s.label(n-4)), true
+		}
+	}
+
+	// Service-before-region form: <resource>.memorydb.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "memorydb" {
+		return aws(model.ProviderAWS, model.ServiceMemoryDB,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+	return model.Match{}, false
+}
+
+// awsResourceAfterRegion matches <resource>.<region>.<service>.amazonaws.com
+// and rejects the bare <service>.<region>.amazonaws.com service endpoint.
+func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) {
+	n := s.n
+	if n < 5 {
+		return model.Match{}, false
+	}
+	resource := s.label(0)
+	if resource == s.label(n-3) {
+		return model.Match{}, false
+	}
+	return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true
+}
+
+func classifyAWSS3(s *scanner) (model.Match, bool) {
+	n := s.n
+
+	// Access points: <ap>.s3-accesspoint[-fips][.dualstack].<region>.amazonaws.com
+	if n >= 5 {
+		svcIdx := -1
+		switch {
+		case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips":
+			svcIdx = n - 4
+		case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint":
+			svcIdx = n - 5
+		}
+		if svcIdx > 0 {
+			res := s.label(0)
+			if res != "s3-accesspoint" && res != "s3-accesspoint-fips" {
+				return aws(model.ProviderAWS, model.ServiceS3,
+					model.PublicSemanticNone, res, s.label(n-3)), true
+			}
+		}
+	}
+
+	// Multi-region access point: <res>.mrap.accesspoint.s3-global.amazonaws.com
+	if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" &&
+		s.label(n-3) == "s3-global" {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), ""), true
+	}
+
+	// S3 Express: <bucket>.s3express-<zone>.<region>.amazonaws.com
+	if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+
+	// S3 website: <bucket>.s3-website.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "s3-website" {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+	// S3 website (region in the service label): <bucket>.s3-website-<region>.amazonaws.com
+	if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), ""), true
+	}
+
+	// Plain S3 without region: <bucket>.s3.amazonaws.com
+	if n == 4 && s.label(n-3) == "s3" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true
+	}
+	// Plain S3 with region: <bucket>.s3.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "s3" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
+	}
+	// Dualstack S3: <bucket>.s3.dualstack.<region>.amazonaws.com
+	if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
+	}
+	return model.Match{}, false
+}

internal/classify/azure.go

@@ -0,0 +1,113 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+func azure(service model.Service, resource, region string) model.Match {
+	return model.Match{
+		Provider:         model.ProviderAzure,
+		Service:          service,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+// classifyAzureCore handles *.blob|dfs|file|queue|table|web.core.windows.net.
+func classifyAzureCore(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "core" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 4)
+	resource := s.label(0)
+	if resource == "" {
+		return model.Match{}, false
+	}
+	switch svc {
+	case "blob", "dfs":
+		return azure(model.ServiceBlobStorage, resource, ""), true
+	case "file":
+		return azure(model.ServiceFileStorage, resource, ""), true
+	case "queue":
+		return azure(model.ServiceQueue, resource, ""), true
+	case "table":
+		return azure(model.ServiceTableStorage, resource, ""), true
+	case "web":
+		return azure(model.ServiceWebApp, resource, ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureDatabase handles <server>.database.windows.net.
+func classifyAzureDatabase(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "database" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceDatabase, s.label(0), ""), true
+}
+
+// classifyAzureFlexibleServer handles <server>.{postgres,mysql}.database.azure.com.
+func classifyAzureFlexibleServer(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "database" {
+		return model.Match{}, false
+	}
+	switch s.label(n - 4) {
+	case "postgres":
+		return azure(model.ServicePostgres, s.label(0), ""), true
+	case "mysql":
+		return azure(model.ServiceMySQL, s.label(0), ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureDocuments handles <account>.documents.azure.com (Cosmos DB).
+func classifyAzureDocuments(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "documents" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceCosmosDB, s.label(0), ""), true
+}
+
+// classifyAzureCosmos handles <account>.{mongo,cassandra,gremlin}.cosmos.azure.com.
+func classifyAzureCosmos(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "cosmos" {
+		return model.Match{}, false
+	}
+	switch s.label(n - 4) {
+	case "mongo", "cassandra", "gremlin":
+		return azure(model.ServiceCosmosDB, s.label(0), ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureRedis handles <name>.redis.cache.windows.net.
+func classifyAzureRedis(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" ||
+		s.label(n-3) != "cache" || s.label(n-4) != "redis" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceRedis, s.label(0), ""), true
+}
+
+// classifyAzureManagedRedis handles <name>.redis.azure.net.
+func classifyAzureManagedRedis(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "azure" || s.label(n-3) != "redis" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceRedis, s.label(0), ""), true
+}
+
+// classifyAzureSearch handles <service>.search.windows.net.
+func classifyAzureSearch(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "search" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceSearch, s.label(0), ""), true
+}

internal/classify/classify.go

@@ -0,0 +1,198 @@
+// Package classify maps a DNS hostname to a cloud/database/storage resource.
+//
+// The hot path is deliberately allocation-free: it never lowercases unless an
+// uppercase byte is present, never splits the hostname into a []string, never
+// uses regexp/JSON/reflection, and returns substrings of the input as the
+// region/resource. See BenchmarkClassify* for the 0 allocs/op assertion.
+package classify
+
+import (
+	"strings"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+const maxLabels = 64
+
+// scanner holds the byte offsets of each dot-separated label of a hostname.
+// It is stack allocated by Classify and never heap allocated.
+type scanner struct {
+	host  string
+	n     int
+	start [maxLabels]int32
+}
+
+func newScanner(host string) scanner {
+	var s scanner
+	s.host = host
+	start := int32(0)
+	for i := 0; i < len(host); i++ {
+		if host[i] == '.' {
+			if s.n >= maxLabels {
+				s.n = -1
+				return s
+			}
+			s.start[s.n] = start
+			s.n++
+			start = int32(i + 1)
+		}
+	}
+	if s.n >= maxLabels {
+		s.n = -1
+		return s
+	}
+	s.start[s.n] = start
+	s.n++
+	return s
+}
+
+// label returns the i-th label (0 = leftmost). Callers must ensure i < n.
+func (s *scanner) label(i int) string {
+	end := int32(len(s.host))
+	if i+1 < s.n {
+		end = s.start[i+1] - 1
+	}
+	return s.host[s.start[i]:end]
+}
+
+func hasUpper(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= 'A' && s[i] <= 'Z' {
+			return true
+		}
+	}
+	return false
+}
+
+func trimTrailingDots(s string) string {
+	for len(s) > 0 && s[len(s)-1] == '.' {
+		s = s[:len(s)-1]
+	}
+	return s
+}
+
+func validHost(host string) bool {
+	if host == "" || len(host) > 253 {
+		return false
+	}
+	for i := 0; i < len(host); i++ {
+		c := host[i]
+		switch {
+		case c >= 'a' && c <= 'z':
+		case c >= '0' && c <= '9':
+		case c == '-' || c == '.' || c == '_':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+// Classify returns the provider resource described by host, if any.
+// The returned Match's string fields alias host.
+func Classify(host string) (model.Match, bool) {
+	host = trimTrailingDots(host)
+	if host == "" || len(host) > 253 {
+		return model.Match{}, false
+	}
+	// Wildcard certificates never describe a concrete resource.
+	if host[0] == '*' {
+		return model.Match{}, false
+	}
+	if hasUpper(host) {
+		host = strings.ToLower(host)
+	}
+	if !validHost(host) {
+		return model.Match{}, false
+	}
+
+	s := newScanner(host)
+	if s.n < 2 {
+		return model.Match{}, false
+	}
+
+	switch {
+	case strings.HasSuffix(host, ".amazonaws.com"):
+		return classifyAWS(&s)
+	case strings.HasSuffix(host, ".core.windows.net"):
+		return classifyAzureCore(&s)
+	case strings.HasSuffix(host, ".database.windows.net"):
+		return classifyAzureDatabase(&s)
+	case strings.HasSuffix(host, ".database.azure.com"):
+		return classifyAzureFlexibleServer(&s)
+	case strings.HasSuffix(host, ".documents.azure.com"):
+		return classifyAzureDocuments(&s)
+	case strings.HasSuffix(host, ".cosmos.azure.com"):
+		return classifyAzureCosmos(&s)
+	case strings.HasSuffix(host, ".cache.windows.net"):
+		return classifyAzureRedis(&s)
+	case strings.HasSuffix(host, ".redis.azure.net"):
+		return classifyAzureManagedRedis(&s)
+	case strings.HasSuffix(host, ".search.windows.net"):
+		return classifyAzureSearch(&s)
+	case strings.HasSuffix(host, ".storage.googleapis.com"):
+		return classifyGCS(&s)
+	case strings.HasSuffix(host, ".firebaseio.com"):
+		return classifyFirebaseIO(&s)
+	case strings.HasSuffix(host, ".firebasedatabase.app"):
+		return classifyFirebaseDB(&s)
+	case strings.HasSuffix(host, ".appspot.com"):
+		return classifyAppspot(&s)
+	case strings.HasSuffix(host, ".r2.cloudflarestorage.com"):
+		return classifyR2(&s)
+	case strings.HasSuffix(host, ".r2.dev"):
+		return classifyR2Dev(&s)
+	case strings.HasSuffix(host, ".digitaloceanspaces.com"):
+		return classifyDigitalOceanSpaces(&s)
+	case strings.HasSuffix(host, ".ondigitalocean.com"):
+		return classifyDigitalOceanDB(&s)
+	case strings.HasSuffix(host, ".supabase.co"):
+		return classifySupabase(&s)
+	case strings.HasSuffix(host, ".supabase.com"):
+		return classifySupabasePooler(&s)
+	case strings.HasSuffix(host, ".customer-oci.com"):
+		return classifyOCIResource(&s)
+	case strings.HasSuffix(host, ".oraclecloud.com"):
+		return classifyOCIGeneric(&s)
+	}
+	return model.Match{}, false
+}
+
+// ShouldEnrich reports whether a non-matching hostname is worth a DNS/CNAME
+// lookup. It rejects wildcards, the certspotter unparsable placeholder, single
+// labels and obvious provider service apexes.
+func ShouldEnrich(host string) bool {
+	host = trimTrailingDots(host)
+	if len(host) < 3 || host[0] == '*' {
+		return false
+	}
+	if host[0] == '<' { // certspotter's "<unparsable>" placeholder
+		return false
+	}
+	if !strings.Contains(host, ".") {
+		return false
+	}
+	if hasUpper(host) {
+		host = strings.ToLower(host)
+	}
+	// Provider service apexes never yield a customer resource via CNAME.
+	switch {
+	case host == "amazonaws.com", strings.HasSuffix(host, ".amazonaws.com"):
+		return false
+	case host == "core.windows.net", strings.HasSuffix(host, ".windows.net"):
+		return false
+	case host == "azure.com", strings.HasSuffix(host, ".azure.com"):
+		return false
+	case host == "googleapis.com", strings.HasSuffix(host, ".googleapis.com"):
+		return false
+	case host == "cloudflarestorage.com", host == "r2.dev":
+		return false
+	case strings.HasSuffix(host, ".digitaloceanspaces.com"):
+		return false
+	case strings.HasSuffix(host, ".supabase.co"), strings.HasSuffix(host, ".supabase.com"):
+		return false
+	case strings.HasSuffix(host, ".oraclecloud.com"), strings.HasSuffix(host, ".customer-oci.com"):
+		return false
+	}
+	return true
+}

internal/classify/classify_test.go

@@ -0,0 +1,243 @@
+package classify
+
+import (
+	"strings"
+	"testing"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+type want struct {
+	provider         model.Provider
+	service          model.Service
+	resource         string
+	region           string
+	resourceSpecific bool
+	semantic         model.PublicSemantic
+}
+
+func run(t *testing.T, host string, w *want) {
+	t.Helper()
+	m, ok := Classify(host)
+	if w == nil {
+		if ok {
+			t.Fatalf("Classify(%q) matched %+v, want no match", host, m)
+		}
+		return
+	}
+	if !ok {
+		t.Fatalf("Classify(%q) did not match, want %+v", host, *w)
+	}
+	if m.Provider != w.provider || m.Service != w.service {
+		t.Fatalf("Classify(%q) = %s/%s, want %s/%s", host, m.Provider, m.Service, w.provider, w.service)
+	}
+	if w.resource != "" && m.Resource != w.resource {
+		t.Fatalf("Classify(%q) resource = %q, want %q", host, m.Resource, w.resource)
+	}
+	if w.region != "" && m.Region != w.region {
+		t.Fatalf("Classify(%q) region = %q, want %q", host, m.Region, w.region)
+	}
+	if m.ResourceSpecific != w.resourceSpecific {
+		t.Fatalf("Classify(%q) resourceSpecific = %v, want %v", host, m.ResourceSpecific, w.resourceSpecific)
+	}
+	if w.semantic != 0 && m.PublicSemantic != w.semantic {
+		t.Fatalf("Classify(%q) semantic = %s, want %s", host, m.PublicSemantic, w.semantic)
+	}
+}
+
+func TestClassifyMatches(t *testing.T) {
+	cases := []struct {
+		host string
+		w    want
+	}{
+		// AWS S3
+		{"bucket.s3.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+		{"bucket.s3.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3.dualstack.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3-website.eu-west-2.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-2", true, 0}},
+		{"bucket.s3-website-us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+		{"ap-123456.s3-accesspoint.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "ap-123456", "us-east-1", true, 0}},
+		{"x.s3-accesspoint.dualstack.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
+		{"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
+		{"x.mrap.accesspoint.s3-global.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "", true, 0}},
+		{"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}},
+		// AWS databases
+		{"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.proxy-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.cluster-abc.us-east-1.docdb.amazonaws.com", want{model.ProviderAWS, model.ServiceDocumentDB, "mydb", "us-east-1", true, 0}},
+		{"cluster.id.eu-central-1.redshift.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
+		{"cluster.id.eu-central-1.redshift-serverless.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
+		{"search-example-abc.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "search-example-abc", "eu-west-1", true, model.PublicSemanticPublicEndpoint}},
+		{"vpc-example.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "vpc-example", "eu-west-1", true, model.PublicSemanticVPCEndpoint}},
+		{"mydb.cluster-abc.us-east-1.neptune.amazonaws.com", want{model.ProviderAWS, model.ServiceNeptune, "mydb", "us-east-1", true, 0}},
+		{"clustercfg.my.xxxx.memorydb.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceMemoryDB, "clustercfg", "us-east-1", true, 0}},
+		{"mycluster.abc.0001.use1.cache.amazonaws.com", want{model.ProviderAWS, model.ServiceElastiCache, "mycluster", "use1", true, 0}},
+		// Azure
+		{"myacct.blob.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
+		{"myacct.dfs.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
+		{"myacct.file.core.windows.net", want{model.ProviderAzure, model.ServiceFileStorage, "myacct", "", true, 0}},
+		{"myacct.queue.core.windows.net", want{model.ProviderAzure, model.ServiceQueue, "myacct", "", true, 0}},
+		{"myacct.table.core.windows.net", want{model.ProviderAzure, model.ServiceTableStorage, "myacct", "", true, 0}},
+		{"myacct.web.core.windows.net", want{model.ProviderAzure, model.ServiceWebApp, "myacct", "", true, 0}},
+		{"myserver.database.windows.net", want{model.ProviderAzure, model.ServiceDatabase, "myserver", "", true, 0}},
+		{"myserver.postgres.database.azure.com", want{model.ProviderAzure, model.ServicePostgres, "myserver", "", true, 0}},
+		{"myserver.mysql.database.azure.com", want{model.ProviderAzure, model.ServiceMySQL, "myserver", "", true, 0}},
+		{"mycosmos.documents.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
+		{"mycosmos.mongo.cosmos.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
+		{"myredis.redis.cache.windows.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
+		{"myredis.redis.azure.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
+		{"mysearch.search.windows.net", want{model.ProviderAzure, model.ServiceSearch, "mysearch", "", true, 0}},
+		// GCP
+		{"bucket.storage.googleapis.com", want{model.ProviderGCP, model.ServiceObjectStorage, "bucket", "", true, 0}},
+		{"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}},
+		{"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}},
+		{"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}},
+		// Cloudflare
+		{"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}},
+		{"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}},
+		// DigitalOcean
+		{"bucket.fra1.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
+		{"bucket.fra1.cdn.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
+		{"mydb.db.ondigitalocean.com", want{model.ProviderDigitalOcean, model.ServiceDatabase, "mydb", "", true, 0}},
+		// Supabase
+		{"abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServiceDatabase, "abcdefgh", "", true, 0}},
+		{"db.abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServicePostgres, "abcdefgh", "", true, 0}},
+		{"aws-0-us-east-1.pooler.supabase.com", want{model.ProviderSupabase, model.ServicePostgres, "aws-0-us-east-1", "", true, 0}},
+		// OCI
+		{"myns.objectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
+		{"myns.swiftobjectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
+		{"objectstorage.us-ashburn-1.oraclecloud.com", want{model.ProviderOracle, model.ServiceObjectStorage, "", "us-ashburn-1", false, 0}},
+		// normalization
+		{"BUCKET.S3.EU-CENTRAL-1.AMAZONAWS.COM", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3.amazonaws.com.", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+	}
+	for _, c := range cases {
+		w := c.w
+		run(t, c.host, &w)
+	}
+}
+
+func TestClassifyRejects(t *testing.T) {
+	rejects := []string{
+		// AWS generic endpoints / non-resources
+		"amazonaws.com",
+		"s3.amazonaws.com",
+		"s3.eu-central-1.amazonaws.com",
+		"s3.dualstack.eu-central-1.amazonaws.com",
+		"s3-accesspoint.us-east-1.amazonaws.com",
+		"dynamodb.eu-central-1.amazonaws.com",
+		"redshift.eu-central-1.amazonaws.com",
+		"rds.us-east-1.amazonaws.com",
+		"docdb.us-east-1.amazonaws.com",
+		"neptune.us-east-1.amazonaws.com",
+		"es.us-east-1.amazonaws.com",
+		"memorydb.us-east-1.amazonaws.com",
+		"cache.amazonaws.com",
+		"use1.cache.amazonaws.com",
+		"s3-global.amazonaws.com",
+		// Azure bare service endpoints
+		"blob.core.windows.net",
+		"database.windows.net",
+		"postgres.database.azure.com",
+		"documents.azure.com",
+		"mongo.cosmos.azure.com",
+		"redis.cache.windows.net",
+		"redis.azure.net",
+		"search.windows.net",
+		// GCP bare service endpoints
+		"storage.googleapis.com",
+		"firebaseio.com",
+		"firebasedatabase.app",
+		"appspot.com",
+		// Cloudflare
+		"r2.cloudflarestorage.com",
+		"r2.dev",
+		// DigitalOcean
+		"fra1.digitaloceanspaces.com",
+		"db.ondigitalocean.com",
+		// Supabase
+		"supabase.co",
+		"pooler.supabase.com",
+		// OCI
+		"oraclecloud.com",
+		"customer-oci.com",
+		// wildcards
+		"*.example.com",
+		"*.blob.core.windows.net",
+		"*.s3.amazonaws.com",
+		// normal hostnames
+		"www.example.com",
+		"foo.bar.example.org",
+		"localhost",
+		"",
+		"-",
+		// invalid / too long
+		"foo bar.com",
+		strings.Repeat("a.", 130) + "com",
+	}
+	for _, host := range rejects {
+		run(t, host, nil)
+	}
+}
+
+func TestShouldEnrich(t *testing.T) {
+	yes := []string{"files.example.com", "db.example.com", "a.b.c.example.org"}
+	no := []string{"*.example.com", "s3.amazonaws.com", "foo.s3.amazonaws.com", "storage.googleapis.com", "<unparsable>", ""}
+	for _, h := range yes {
+		if !ShouldEnrich(h) {
+			t.Errorf("ShouldEnrich(%q) = false, want true", h)
+		}
+	}
+	for _, h := range no {
+		if ShouldEnrich(h) {
+			t.Errorf("ShouldEnrich(%q) = true, want false", h)
+		}
+	}
+}
+
+var benchInputs = []string{
+	"www.example.com",
+	"foo.bar.example.org",
+	"bucket.s3.eu-central-1.amazonaws.com",
+	"foo.blob.core.windows.net",
+	"db.example.supabase.co",
+	"project.firebaseio.com",
+	"search-example-abc.eu-west-1.es.amazonaws.com",
+}
+
+func BenchmarkClassifyAWS(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("bucket.s3.eu-central-1.amazonaws.com")
+	}
+}
+
+func BenchmarkClassifyAzure(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("foo.blob.core.windows.net")
+	}
+}
+
+func BenchmarkRejectNormalDomain(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("www.example.com")
+	}
+}
+
+func BenchmarkClassifyMixed(b *testing.B) {
+	allocs := testing.AllocsPerRun(1000, func() {
+		for _, host := range benchInputs {
+			Classify(host)
+		}
+	})
+	if allocs != 0 {
+		b.Fatalf("Classify allocates %.2f allocs/op, want 0", allocs)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, host := range benchInputs {
+			Classify(host)
+		}
+	}
+}

internal/classify/cloudflare.go

@@ -0,0 +1,34 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyR2 handles <account-id>.r2.cloudflarestorage.com.
+func classifyR2(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "cloudflarestorage" || s.label(n-3) != "r2" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderCloudflare,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}
+
+// classifyR2Dev handles <account>.r2.dev.  The hostname form is explicitly a
+// public development/public URL, which is why it carries PublicSemanticPublicURL
+// while still not asserting that data is readable.
+func classifyR2Dev(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "dev" || s.label(n-2) != "r2" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderCloudflare,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		PublicSemantic:   model.PublicSemanticPublicURL,
+		Resource:         s.label(0),
+	}, true
+}

internal/classify/digitalocean.go

@@ -0,0 +1,39 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyDigitalOceanSpaces handles
+// <bucket>.<region>.digitaloceanspaces.com and the cdn variant.
+func classifyDigitalOceanSpaces(s *scanner) (model.Match, bool) {
+	n := s.n
+	if s.label(n-1) != "com" || s.label(n-2) != "digitaloceanspaces" {
+		return model.Match{}, false
+	}
+	// <bucket>.<region>.digitaloceanspaces.com (n==4) or the cdn variant (n==5)
+	if n != 4 && !(n == 5 && s.label(2) == "cdn") {
+		return model.Match{}, false
+	}
+	bucket := s.label(0)
+	region := s.label(1)
+	return model.Match{
+		Provider:         model.ProviderDigitalOcean,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         bucket,
+	}, true
+}
+
+// classifyDigitalOceanDB handles <name>.db.ondigitalocean.com.
+func classifyDigitalOceanDB(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "ondigitalocean" || s.label(n-3) != "db" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderDigitalOcean,
+		Service:          model.ServiceDatabase,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}

internal/classify/gcp.go

@@ -0,0 +1,49 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+func gcp(service model.Service, resource, region string) model.Match {
+	return model.Match{
+		Provider:         model.ProviderGCP,
+		Service:          service,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+// classifyGCS handles <bucket>.storage.googleapis.com.
+func classifyGCS(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "googleapis" || s.label(n-3) != "storage" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceObjectStorage, s.label(0), ""), true
+}
+
+// classifyFirebaseIO handles <database>.firebaseio.com.
+func classifyFirebaseIO(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "firebaseio" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceFirebaseDB, s.label(0), ""), true
+}
+
+// classifyFirebaseDB handles <database>.<region>.firebasedatabase.app.
+func classifyFirebaseDB(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "app" || s.label(n-2) != "firebasedatabase" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceFirebaseDB, s.label(0), s.label(1)), true
+}
+
+// classifyAppspot handles <project>.appspot.com.
+func classifyAppspot(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "appspot" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceWebApp, s.label(0), ""), true
+}

internal/classify/oracle.go

@@ -0,0 +1,44 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyOCIResource handles
+// <namespace>.objectstorage.<region>.oci.customer-oci.com and the swift variant.
+// The namespace is in the hostname, so it is resource specific.
+func classifyOCIResource(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 6 || s.label(n-1) != "com" || s.label(n-2) != "customer-oci" || s.label(n-3) != "oci" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 5)
+	if svc != "objectstorage" && svc != "swiftobjectstorage" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderOracle,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Region:           s.label(n - 4),
+		Resource:         s.label(0),
+	}, true
+}
+
+// classifyOCIGeneric handles objectstorage.<region>.oraclecloud.com. The bucket
+// name lives in the URL path rather than the hostname, so resource_specific is
+// false and these normally do not become findings.
+func classifyOCIGeneric(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "oraclecloud" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 4)
+	if svc != "objectstorage" && svc != "swiftobjectstorage" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderOracle,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: false,
+		Region:           s.label(n - 3),
+	}, true
+}

internal/classify/supabase.go

@@ -0,0 +1,45 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifySupabase handles <project-ref>.supabase.co and
+// db.<project-ref>.supabase.co (the latter classified as PostgreSQL).
+func classifySupabase(s *scanner) (model.Match, bool) {
+	n := s.n
+	if s.label(n-1) != "co" || s.label(n-2) != "supabase" {
+		return model.Match{}, false
+	}
+	switch n {
+	case 3:
+		return model.Match{
+			Provider:         model.ProviderSupabase,
+			Service:          model.ServiceDatabase,
+			ResourceSpecific: true,
+			Resource:         s.label(0),
+		}, true
+	case 4:
+		if s.label(0) == "db" {
+			return model.Match{
+				Provider:         model.ProviderSupabase,
+				Service:          model.ServicePostgres,
+				ResourceSpecific: true,
+				Resource:         s.label(1),
+			}, true
+		}
+	}
+	return model.Match{}, false
+}
+
+// classifySupabasePooler handles aws-<index>-<region>.pooler.supabase.com.
+func classifySupabasePooler(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "supabase" || s.label(n-3) != "pooler" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderSupabase,
+		Service:          model.ServicePostgres,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}

internal/config/config.go

@@ -0,0 +1,160 @@
+// Package config loads all ctcloud configuration from environment variables.
+// There are no configuration files at runtime.
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Config struct {
+	DatabaseURL string
+	DBMaxConns  int
+
+	DNSServers       []string
+	DNSWorkers       int
+	DNSTimeout       time.Duration
+	DNSCacheSize     int
+	DNSMaxCNAMEdepth int
+	DNSPositiveTTL   time.Duration
+	DNSNegativeTTL   time.Duration
+
+	DirectQueueSize int
+	DNSQueueSize    int
+	DBQueueSize     int
+
+	DBBatchSize     int
+	DBFlushInterval time.Duration
+
+	ActiveProbing bool
+	ProbeWorkers  int
+	ProbeTimeout  time.Duration
+
+	MetricsAddr   string
+	LogLevel      string
+	StatsInterval time.Duration
+
+	LogListSource   string
+	StartAtEnd      bool
+	ShutdownTimeout time.Duration
+	DedupeTTL       time.Duration
+}
+
+// DefaultLogListSource is the public CT log list used by certspotter.
+const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.json"
+
+// DefaultDNSServers are load-balanced across. Set DNS_SERVERS=system to use the
+// host resolver instead, or provide your own comma-separated list.
+var DefaultDNSServers = []string{"1.1.1.1:53", "8.8.8.8:53", "1.0.0.1:53"}
+
+func Load() (*Config, error) {
+	c := &Config{
+		DatabaseURL:      os.Getenv("DATABASE_URL"),
+		DBMaxConns:       envInt("DB_MAX_CONNS", 8),
+		DNSServers:       envList("DNS_SERVERS", DefaultDNSServers),
+		DNSWorkers:       envInt("DNS_WORKERS", 16),
+		DNSTimeout:       envDuration("DNS_TIMEOUT", 5*time.Second),
+		DNSCacheSize:     envInt("DNS_CACHE_SIZE", 1<<20),
+		DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
+		DNSPositiveTTL:   envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
+		DNSNegativeTTL:   envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
+		DirectQueueSize:  envInt("DIRECT_QUEUE_SIZE", 8192),
+		DNSQueueSize:     envInt("DNS_QUEUE_SIZE", 65536),
+		DBQueueSize:      envInt("DB_QUEUE_SIZE", 8192),
+		DBBatchSize:      envInt("DB_BATCH_SIZE", 500),
+		DBFlushInterval:  envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
+		ActiveProbing:    envBool("ACTIVE_PROBING", false),
+		ProbeWorkers:     envInt("PROBE_WORKERS", 4),
+		ProbeTimeout:     envDuration("PROBE_TIMEOUT", 5*time.Second),
+		MetricsAddr:      envString("METRICS_ADDR", ":9090"),
+		LogLevel:         envString("LOG_LEVEL", "info"),
+		StatsInterval:    envDuration("STATS_INTERVAL", 60*time.Second),
+		LogListSource:    envString("LOG_LIST_SOURCE", DefaultLogListSource),
+		StartAtEnd:       envBool("START_AT_END", false),
+		ShutdownTimeout:  envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		DedupeTTL:        envDuration("DEDUP_TTL", time.Hour),
+	}
+	if err := c.validate(); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+func (c *Config) validate() error {
+	if c.DatabaseURL == "" {
+		return fmt.Errorf("DATABASE_URL is required")
+	}
+	if c.DNSWorkers < 1 {
+		return fmt.Errorf("DNS_WORKERS must be >= 1")
+	}
+	if c.DNSMaxCNAMEdepth < 1 {
+		return fmt.Errorf("DNS_MAX_CNAME_DEPTH must be >= 1")
+	}
+	if c.DirectQueueSize < 1 || c.DNSQueueSize < 1 || c.DBQueueSize < 1 {
+		return fmt.Errorf("queue sizes must be >= 1")
+	}
+	if c.DBBatchSize < 1 {
+		return fmt.Errorf("DB_BATCH_SIZE must be >= 1")
+	}
+	switch c.LogLevel {
+	case "debug", "info", "warn", "error":
+	default:
+		return fmt.Errorf("LOG_LEVEL must be one of debug, info, warn, error")
+	}
+	return nil
+}
+
+func envList(key string, def []string) []string {
+	v := os.Getenv(key)
+	if v == "" {
+		return def
+	}
+	parts := strings.Split(v, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	if len(out) == 0 {
+		return def
+	}
+	return out
+}
+
+func envString(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func envInt(key string, def int) int {
+	if v := os.Getenv(key); v != "" {
+		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+			return n
+		}
+	}
+	return def
+}
+
+func envBool(key string, def bool) bool {
+	if v := os.Getenv(key); v != "" {
+		if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+			return b
+		}
+	}
+	return def
+}
+
+func envDuration(key string, def time.Duration) time.Duration {
+	if v := os.Getenv(key); v != "" {
+		if d, err := time.ParseDuration(strings.TrimSpace(v)); err == nil {
+			return d
+		}
+	}
+	return def
+}

internal/config/config_test.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+func TestLoadDefaults(t *testing.T) {
+	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	c, err := Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.DatabaseURL != "postgres://localhost/ctcloud" {
+		t.Fatalf("database url = %q", c.DatabaseURL)
+	}
+	if c.DNSWorkers != 16 || c.DNSMaxCNAMEdepth != 8 {
+		t.Fatalf("unexpected DNS defaults: %+v", c)
+	}
+	if c.ActiveProbing {
+		t.Fatal("active probing must default to false")
+	}
+}
+
+func TestLoadOverrides(t *testing.T) {
+	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("DNS_WORKERS", "4")
+	t.Setenv("ACTIVE_PROBING", "true")
+	t.Setenv("DB_FLUSH_INTERVAL", "250ms")
+	t.Setenv("STATS_INTERVAL", "30s")
+	c, err := Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.DNSWorkers != 4 || !c.ActiveProbing {
+		t.Fatalf("overrides not applied: %+v", c)
+	}
+	if c.DBFlushInterval != 250*time.Millisecond || c.StatsInterval != 30*time.Second {
+		t.Fatalf("duration overrides not applied: %+v", c)
+	}
+}
+
+func TestLoadInvalid(t *testing.T) {
+	t.Setenv("DATABASE_URL", "")
+	if _, err := Load(); err == nil {
+		t.Fatal("expected error without DATABASE_URL")
+	}
+	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("LOG_LEVEL", "verbose")
+	if _, err := Load(); err == nil {
+		t.Fatal("expected error for bad LOG_LEVEL")
+	}
+}

internal/dns/cache.go

@@ -0,0 +1,99 @@
+package dns
+
+import (
+	"sync"
+	"time"
+)
+
+type cacheEntry[V any] struct {
+	val V
+	exp time.Time
+}
+
+type shard[V any] struct {
+	mu sync.Mutex
+	m  map[string]cacheEntry[V]
+}
+
+// TTLCache is a sharded, memory-bounded, in-memory TTL cache. It is used for
+// positive DNS results, negative DNS results, CNAME targets and hostname dedup.
+//
+// ponytail: once a shard is full, an arbitrary entry is evicted rather than the
+// least-recently-used one. Swap in an LRU only if the hit rate actually matters.
+type TTLCache[V any] struct {
+	shards   []shard[V]
+	mask     uint64
+	maxShard int
+}
+
+func NewTTLCache[V any](maxEntries, shards int) *TTLCache[V] {
+	if shards < 1 {
+		shards = 1
+	}
+	n := 1
+	for n < shards {
+		n <<= 1
+	}
+	maxShard := maxEntries / n
+	if maxShard < 1 {
+		maxShard = 1
+	}
+	c := &TTLCache[V]{mask: uint64(n - 1), maxShard: maxShard}
+	c.shards = make([]shard[V], n)
+	for i := range c.shards {
+		c.shards[i].m = make(map[string]cacheEntry[V], maxShard)
+	}
+	return c
+}
+
+func hashString(s string) uint64 {
+	const prime = 1099511628211
+	h := uint64(14695981039346656037)
+	for i := 0; i < len(s); i++ {
+		h ^= uint64(s[i])
+		h *= prime
+	}
+	return h
+}
+
+func (c *TTLCache[V]) Get(key string) (V, bool) {
+	s := &c.shards[hashString(key)&c.mask]
+	s.mu.Lock()
+	e, ok := s.m[key]
+	if ok && time.Now().Before(e.exp) {
+		s.mu.Unlock()
+		return e.val, true
+	}
+	if ok {
+		delete(s.m, key)
+	}
+	s.mu.Unlock()
+	var zero V
+	return zero, false
+}
+
+func (c *TTLCache[V]) Set(key string, val V, ttl time.Duration) {
+	if ttl <= 0 {
+		return
+	}
+	s := &c.shards[hashString(key)&c.mask]
+	s.mu.Lock()
+	if len(s.m) >= c.maxShard {
+		for k := range s.m {
+			delete(s.m, k)
+			break
+		}
+	}
+	s.m[key] = cacheEntry[V]{val: val, exp: time.Now().Add(ttl)}
+	s.mu.Unlock()
+}
+
+func (c *TTLCache[V]) Len() int {
+	total := 0
+	for i := range c.shards {
+		c.shards[i].mu.Lock()
+		total += len(c.shards[i].m)
+		c.shards[i].mu.Unlock()
+	}
+	return total
+}

internal/dns/pool.go

@@ -0,0 +1,119 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"sync/atomic"
+	"time"
+)
+
+// ServerPool load-balances DNS lookups across multiple resolvers using
+// round-robin, and fails over to the next server on a timeout or server error.
+// Definitive answers (NXDOMAIN) are returned immediately and are not retried on
+// every server.
+// defaultAttemptTimeout bounds each individual server attempt so a dead
+// resolver costs a couple of seconds before failover, not the entire lookup
+// budget.
+const defaultAttemptTimeout = 2 * time.Second
+
+type ServerPool struct {
+	resolvers      []Resolver
+	next           atomic.Uint64
+	attemptTimeout time.Duration
+}
+
+func (p *ServerPool) withAttemptTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
+	t := p.attemptTimeout
+	if t <= 0 {
+		t = defaultAttemptTimeout
+	}
+	return context.WithTimeout(ctx, t)
+}
+
+// NewServerPool builds a pool from "host", "host:port" or "system" entries.
+// An empty list uses the system resolver.
+func NewServerPool(servers []string) *ServerPool {
+	var resolvers []Resolver
+	for _, s := range servers {
+		s = strings.TrimSpace(s)
+		switch {
+		case s == "":
+			continue
+		case s == "system" || s == "local":
+			return &ServerPool{resolvers: []Resolver{&GoResolver{R: net.DefaultResolver}}}
+		default:
+			if !strings.Contains(s, ":") {
+				s += ":53"
+			}
+			resolvers = append(resolvers, &GoResolver{R: newServerResolver(s)})
+		}
+	}
+	if len(resolvers) == 0 {
+		resolvers = append(resolvers, &GoResolver{R: net.DefaultResolver})
+	}
+	return &ServerPool{resolvers: resolvers}
+}
+
+func newServerResolver(server string) *net.Resolver {
+	return &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
+			if network != "tcp" {
+				network = "udp"
+			}
+			d := net.Dialer{Timeout: 5 * time.Second}
+			return d.DialContext(ctx, network, server)
+		},
+	}
+}
+
+func (p *ServerPool) pick() int {
+	return int((p.next.Add(1) - 1) % uint64(len(p.resolvers)))
+}
+
+// definitive reports whether err is a real DNS answer (NXDOMAIN) that should not
+// be retried against other servers.
+func definitive(err error) bool {
+	var dnsErr *net.DNSError
+	return errors.As(err, &dnsErr) && dnsErr.IsNotFound
+}
+
+func (p *ServerPool) LookupCNAME(ctx context.Context, host string) (string, error) {
+	n := len(p.resolvers)
+	start := p.pick()
+	var last error
+	for i := 0; i < n; i++ {
+		actx, cancel := p.withAttemptTimeout(ctx)
+		cname, err := p.resolvers[(start+i)%n].LookupCNAME(actx, host)
+		cancel()
+		if err == nil {
+			return cname, nil
+		}
+		if definitive(err) {
+			return "", err
+		}
+		last = err
+	}
+	return "", last
+}
+
+func (p *ServerPool) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	n := len(p.resolvers)
+	start := p.pick()
+	var last error
+	for i := 0; i < n; i++ {
+		actx, cancel := p.withAttemptTimeout(ctx)
+		ips, err := p.resolvers[(start+i)%n].LookupIP(actx, host)
+		cancel()
+		if err == nil {
+			return ips, nil
+		}
+		if definitive(err) {
+			return nil, err
+		}
+		last = err
+	}
+	return nil, last
+}

internal/dns/pool_test.go

@@ -0,0 +1,68 @@
+package dns
+
+import (
+	"context"
+	"net"
+	"sync/atomic"
+	"testing"
+)
+
+type fakeLookup struct {
+	calls    atomic.Int32
+	cnameErr error
+	ipErr    error
+}
+
+func (f *fakeLookup) LookupCNAME(ctx context.Context, host string) (string, error) {
+	f.calls.Add(1)
+	if f.cnameErr != nil {
+		return "", f.cnameErr
+	}
+	return host, nil
+}
+
+func (f *fakeLookup) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	f.calls.Add(1)
+	if f.ipErr != nil {
+		return nil, f.ipErr
+	}
+	return []net.IP{net.ParseIP("1.2.3.4")}, nil
+}
+
+func TestServerPoolRoundRobin(t *testing.T) {
+	a, b := &fakeLookup{}, &fakeLookup{}
+	p := &ServerPool{resolvers: []Resolver{a, b}}
+	for i := 0; i < 4; i++ {
+		if _, err := p.LookupIP(context.Background(), "example.com"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if a.calls.Load() != 2 || b.calls.Load() != 2 {
+		t.Fatalf("round robin calls: a=%d b=%d, want 2/2", a.calls.Load(), b.calls.Load())
+	}
+}
+
+func TestServerPoolFailover(t *testing.T) {
+	a := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}}
+	b := &fakeLookup{}
+	p := &ServerPool{resolvers: []Resolver{a, b}}
+	ips, err := p.LookupIP(context.Background(), "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ips) != 1 || a.calls.Load() != 1 {
+		t.Fatalf("expected failover to second server, a.calls=%d", a.calls.Load())
+	}
+}
+
+func TestServerPoolNoFailoverOnNXDOMAIN(t *testing.T) {
+	a := &fakeLookup{ipErr: &net.DNSError{Err: "no such host", IsNotFound: true}}
+	b := &fakeLookup{}
+	p := &ServerPool{resolvers: []Resolver{a, b}}
+	if _, err := p.LookupIP(context.Background(), "missing.example.com"); err == nil {
+		t.Fatal("expected error")
+	}
+	if b.calls.Load() != 0 {
+		t.Fatalf("NXDOMAIN must not fail over; b.calls=%d", b.calls.Load())
+	}
+}

internal/dns/resolver.go

@@ -0,0 +1,131 @@
+// Package dns provides a bounded, cached DNS enricher that follows CNAME chains
+// with loop detection.
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+)
+
+// ErrCNAMELoop is returned when a CNAME chain revisits a host.
+var ErrCNAMELoop = errors.New("cname loop detected")
+
+// Resolver is the subset of DNS operations the enricher needs.
+type Resolver interface {
+	LookupCNAME(ctx context.Context, host string) (string, error)
+	LookupIP(ctx context.Context, host string) ([]net.IP, error)
+}
+
+// Result is a resolved hostname: its CNAME chain (chain[0] is the queried host)
+// and any A/AAAA addresses.
+type Result struct {
+	Chain []string
+	IPs   []net.IP
+}
+
+// GoResolver uses the standard library resolver.
+//
+// ponytail: net.Resolver.LookupCNAME returns the final canonical name, so the
+// chain has at most two entries and intermediate CNAME hops are not observed.
+// Swap in miekg/dns if the full chain is ever needed.
+type GoResolver struct {
+	R *net.Resolver
+}
+
+func NewGoResolver() *GoResolver {
+	return &GoResolver{R: net.DefaultResolver}
+}
+
+func (g *GoResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	return g.R.LookupCNAME(ctx, host)
+}
+
+func (g *GoResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	return g.R.LookupIP(ctx, "ip", host)
+}
+
+// Follow walks a CNAME chain up to maxDepth hops, detecting loops.
+func Follow(ctx context.Context, r Resolver, host string, maxDepth int) ([]string, error) {
+	if maxDepth < 1 {
+		maxDepth = 1
+	}
+	chain := []string{host}
+	cur := host
+	for i := 0; i < maxDepth; i++ {
+		next, err := r.LookupCNAME(ctx, cur)
+		if err != nil {
+			return chain, err
+		}
+		next = strings.TrimSuffix(next, ".")
+		if next == "" || next == cur {
+			break
+		}
+		for _, seen := range chain {
+			if seen == next {
+				return chain, ErrCNAMELoop
+			}
+		}
+		chain = append(chain, next)
+		cur = next
+	}
+	return chain, nil
+}
+
+// Enricher resolves hostnames with caching and bounded CNAME depth. It is safe
+// for concurrent use.
+type Enricher struct {
+	Resolver    Resolver
+	Cache       *TTLCache[Result]
+	PositiveTTL time.Duration
+	NegativeTTL time.Duration
+	Timeout     time.Duration
+	MaxDepth    int
+	Metrics     *metrics.Metrics
+}
+
+// Resolve returns the result for host. ok is false when the lookup failed;
+// failures (including "no such host") are negatively cached for NegativeTTL.
+func (e *Enricher) Resolve(ctx context.Context, host string) (Result, bool) {
+	if e.Timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, e.Timeout)
+		defer cancel()
+	}
+	if e.Cache != nil {
+		if res, ok := e.Cache.Get(host); ok {
+			if e.Metrics != nil {
+				e.Metrics.DNSCacheHits.Add(1)
+			}
+			return res, len(res.Chain) > 0 || len(res.IPs) > 0
+		}
+	}
+	if e.Metrics != nil {
+		e.Metrics.DNSQueries.Add(1)
+	}
+
+	chain, cnameErr := Follow(ctx, e.Resolver, host, e.MaxDepth)
+	if cnameErr != nil && len(chain) == 1 {
+		// A definitive NXDOMAIN has no A/AAAA to fetch either, so skip the
+		// second lookup. NXDOMAIN is a valid answer, not an infrastructure error.
+		if e.Metrics != nil && !definitive(cnameErr) {
+			e.Metrics.DNSErrors.Add(1)
+		}
+		if e.Cache != nil {
+			e.Cache.Set(host, Result{}, e.NegativeTTL)
+		}
+		return Result{}, false
+	}
+	res := Result{Chain: chain}
+	if ips, err := e.Resolver.LookupIP(ctx, host); err == nil {
+		res.IPs = ips
+	}
+	if e.Cache != nil {
+		e.Cache.Set(host, res, e.PositiveTTL)
+	}
+	return res, true
+}

internal/dns/resolver_test.go

@@ -0,0 +1,142 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"testing"
+	"time"
+)
+
+type fakeResolver struct {
+	cnames map[string]string
+	ips    map[string][]net.IP
+	err    error
+}
+
+func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	if f.err != nil {
+		return "", f.err
+	}
+	if c, ok := f.cnames[host]; ok {
+		return c, nil
+	}
+	return host, nil
+}
+
+func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	if f.err != nil {
+		return nil, f.err
+	}
+	return f.ips[host], nil
+}
+
+func TestFollowChain(t *testing.T) {
+	r := &fakeResolver{cnames: map[string]string{
+		"a.example.com":      "b.example.com",
+		"b.example.com":      "c.s3.amazonaws.com",
+		"c.s3.amazonaws.com": "c.s3.amazonaws.com",
+	}}
+	chain, err := Follow(context.Background(), r, "a.example.com", 8)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"a.example.com", "b.example.com", "c.s3.amazonaws.com"}
+	if len(chain) != len(want) {
+		t.Fatalf("chain = %v, want %v", chain, want)
+	}
+	for i := range want {
+		if chain[i] != want[i] {
+			t.Fatalf("chain = %v, want %v", chain, want)
+		}
+	}
+}
+
+func TestFollowLoop(t *testing.T) {
+	r := &fakeResolver{cnames: map[string]string{
+		"a.example.com": "b.example.com",
+		"b.example.com": "a.example.com",
+	}}
+	chain, err := Follow(context.Background(), r, "a.example.com", 8)
+	if !errors.Is(err, ErrCNAMELoop) {
+		t.Fatalf("err = %v, want ErrCNAMELoop", err)
+	}
+	if len(chain) < 2 {
+		t.Fatalf("chain = %v, want at least 2 entries", chain)
+	}
+}
+
+func TestFollowDepth(t *testing.T) {
+	r := &fakeResolver{cnames: map[string]string{
+		"h1.example.com": "h2.example.com",
+		"h2.example.com": "h3.example.com",
+		"h3.example.com": "h4.example.com",
+	}}
+	chain, err := Follow(context.Background(), r, "h1.example.com", 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(chain) != 3 {
+		t.Fatalf("chain = %v, want 3 entries (host + 2 hops)", chain)
+	}
+}
+
+func TestEnricherCaches(t *testing.T) {
+	r := &fakeResolver{
+		cnames: map[string]string{"a.example.com": "b.s3.amazonaws.com"},
+		ips:    map[string][]net.IP{"a.example.com": {net.ParseIP("1.2.3.4")}},
+	}
+	e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
+	if _, ok := e.Resolve(context.Background(), "a.example.com"); !ok {
+		t.Fatal("first resolve failed")
+	}
+	r.cnames = nil // a cached lookup must not hit the resolver
+	res, ok := e.Resolve(context.Background(), "a.example.com")
+	if !ok || len(res.IPs) != 1 {
+		t.Fatalf("cached resolve = %+v ok=%v", res, ok)
+	}
+}
+
+func TestEnricherNegativeCache(t *testing.T) {
+	r := &fakeResolver{err: &net.DNSError{Err: "no such host", IsNotFound: true}}
+	e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
+	if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok {
+		t.Fatal("expected failure")
+	}
+	if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok {
+		t.Fatal("expected cached failure")
+	}
+}
+
+type ctxResolver struct{}
+
+func (ctxResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	return "", ctx.Err()
+}
+func (ctxResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	return nil, ctx.Err()
+}
+
+func TestResolveContextCancellation(t *testing.T) {
+	e := &Enricher{Resolver: ctxResolver{}, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, ok := e.Resolve(ctx, "example.com"); ok {
+		t.Fatal("expected cancellation to fail resolution")
+	}
+}
+
+func TestCacheBoundAndExpiry(t *testing.T) {
+	c := NewTTLCache[int](8, 2)
+	for i := 0; i < 100; i++ {
+		c.Set(string(rune('a'+i%26))+string(rune('0'+i%10)), i, time.Minute)
+	}
+	if c.Len() > 8 {
+		t.Fatalf("cache len = %d, want <= 8", c.Len())
+	}
+	c.Set("k", 1, time.Millisecond)
+	time.Sleep(5 * time.Millisecond)
+	if _, ok := c.Get("k"); ok {
+		t.Fatal("expected expired entry to be gone")
+	}
+}

internal/metrics/metrics.go

@@ -0,0 +1,91 @@
+// Package metrics exposes lightweight Prometheus text-format metrics without
+// pulling in a metrics dependency. Counters are plain atomics; gauges are
+// callback functions so they always report current values.
+package metrics
+
+import (
+	"fmt"
+	"net/http"
+	"runtime"
+	"sync/atomic"
+)
+
+type gauge struct {
+	name  string
+	help  string
+	extra string
+	fn    func() float64
+}
+
+type Metrics struct {
+	CTCertificates atomic.Uint64
+	CTIdentifiers  atomic.Uint64
+
+	DirectMatches atomic.Uint64
+	CNAMEMatches  atomic.Uint64
+
+	DNSQueries   atomic.Uint64
+	DNSCacheHits atomic.Uint64
+	DNSErrors    atomic.Uint64
+
+	Candidates atomic.Uint64
+
+	DBRows   atomic.Uint64
+	DBErrors atomic.Uint64
+
+	QueueDrops    atomic.Uint64
+	QueueFull     atomic.Uint64
+	MalformedCT   atomic.Uint64
+	CTErrors      atomic.Uint64
+	ProbeAttempts atomic.Uint64
+	ProbePublic   atomic.Uint64
+
+	gauges []gauge
+}
+
+func New() *Metrics {
+	m := &Metrics{}
+	m.gauges = []gauge{
+		{"go_goroutines", "Number of goroutines", "", func() float64 { return float64(runtime.NumGoroutine()) }},
+		{"process_system_bytes", "Bytes of memory obtained from the OS", "", func() float64 {
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+			return float64(ms.Sys)
+		}},
+	}
+	return m
+}
+
+// SetQueueDepth registers a gauge reporting the live depth of a named queue.
+func (m *Metrics) SetQueueDepth(name string, fn func() float64) {
+	m.gauges = append(m.gauges, gauge{"queue_depth", "Current depth of an internal queue", `{queue="` + name + `"}`, fn})
+}
+
+// Handler returns an http.Handler serving the Prometheus text exposition format.
+func (m *Metrics) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+		counter := func(name, help string, v uint64) {
+			fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v)
+		}
+		counter("ct_certificates_total", "Certificates processed from CT logs", m.CTCertificates.Load())
+		counter("ct_identifiers_total", "DNS identifiers parsed from CT certificates", m.CTIdentifiers.Load())
+		counter("classifier_direct_matches_total", "Identifiers matching a provider directly", m.DirectMatches.Load())
+		counter("classifier_cname_matches_total", "CNAME targets matching a provider", m.CNAMEMatches.Load())
+		counter("dns_queries_total", "DNS queries performed", m.DNSQueries.Load())
+		counter("dns_cache_hits_total", "DNS lookups served from cache", m.DNSCacheHits.Load())
+		counter("dns_errors_total", "DNS lookup errors", m.DNSErrors.Load())
+		counter("candidates_total", "Provider candidates produced", m.Candidates.Load())
+		counter("db_rows_total", "Rows written to PostgreSQL", m.DBRows.Load())
+		counter("db_errors_total", "PostgreSQL write errors", m.DBErrors.Load())
+		counter("queue_drops_total", "Items dropped because a queue was full", m.QueueDrops.Load())
+		counter("queue_full_total", "Times a queue was observed full", m.QueueFull.Load())
+		counter("ct_malformed_entries_total", "Malformed CT log entries", m.MalformedCT.Load())
+		counter("ct_errors_total", "Non-fatal CT errors", m.CTErrors.Load())
+		counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
+		counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
+		for _, g := range m.gauges {
+			fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
+		}
+	})
+}

internal/model/model.go

@@ -0,0 +1,219 @@
+// Package model holds the compact enums and hot-path structs shared by the
+// classifier, pipeline and store.
+package model
+
+// Provider identifies a cloud/database/storage provider.
+type Provider uint8
+
+const (
+	ProviderUnknown Provider = iota
+	ProviderAWS
+	ProviderAzure
+	ProviderGCP
+	ProviderCloudflare
+	ProviderDigitalOcean
+	ProviderSupabase
+	ProviderOracle
+)
+
+func (p Provider) String() string {
+	switch p {
+	case ProviderAWS:
+		return "aws"
+	case ProviderAzure:
+		return "azure"
+	case ProviderGCP:
+		return "gcp"
+	case ProviderCloudflare:
+		return "cloudflare"
+	case ProviderDigitalOcean:
+		return "digitalocean"
+	case ProviderSupabase:
+		return "supabase"
+	case ProviderOracle:
+		return "oracle"
+	default:
+		return "unknown"
+	}
+}
+
+// Service identifies the provider service the resource belongs to.
+type Service uint8
+
+const (
+	ServiceUnknown Service = iota
+
+	ServiceObjectStorage
+	ServiceS3
+	ServiceBlobStorage
+	ServiceDatabase
+	ServicePostgres
+	ServiceMySQL
+	ServiceRDS
+	ServiceDocumentDB
+	ServiceRedshift
+	ServiceOpenSearch
+	ServiceNeptune
+	ServiceRedis
+	ServiceMemoryDB
+	ServiceCosmosDB
+	ServiceFirebaseDB
+	ServiceElastiCache
+	ServiceSearch
+	ServiceQueue
+	ServiceFileStorage
+	ServiceTableStorage
+	ServiceWebApp
+)
+
+func (s Service) String() string {
+	switch s {
+	case ServiceObjectStorage:
+		return "object_storage"
+	case ServiceS3:
+		return "s3"
+	case ServiceBlobStorage:
+		return "blob_storage"
+	case ServiceDatabase:
+		return "database"
+	case ServicePostgres:
+		return "postgres"
+	case ServiceMySQL:
+		return "mysql"
+	case ServiceRDS:
+		return "rds"
+	case ServiceDocumentDB:
+		return "documentdb"
+	case ServiceRedshift:
+		return "redshift"
+	case ServiceOpenSearch:
+		return "opensearch"
+	case ServiceNeptune:
+		return "neptune"
+	case ServiceRedis:
+		return "redis"
+	case ServiceMemoryDB:
+		return "memorydb"
+	case ServiceCosmosDB:
+		return "cosmosdb"
+	case ServiceFirebaseDB:
+		return "firebase_db"
+	case ServiceElastiCache:
+		return "elasticache"
+	case ServiceSearch:
+		return "search"
+	case ServiceQueue:
+		return "queue"
+	case ServiceFileStorage:
+		return "file_storage"
+	case ServiceTableStorage:
+		return "table_storage"
+	case ServiceWebApp:
+		return "web_app"
+	default:
+		return "unknown"
+	}
+}
+
+// Source is where a candidate was discovered.
+type Source uint8
+
+const (
+	SourceUnknown Source = iota
+	SourceDirectCT
+	SourceCNAME
+	SourceDNSAlias
+)
+
+func (s Source) String() string {
+	switch s {
+	case SourceDirectCT:
+		return "direct_ct"
+	case SourceCNAME:
+		return "cname"
+	case SourceDNSAlias:
+		return "dns_alias"
+	default:
+		return "unknown"
+	}
+}
+
+// PublicSemantic captures when the hostname format itself carries public
+// accessibility meaning. It never asserts that data is actually exposed.
+type PublicSemantic uint8
+
+const (
+	PublicSemanticNone PublicSemantic = iota
+	PublicSemanticPublicEndpoint
+	PublicSemanticVPCEndpoint
+	PublicSemanticPublicURL
+)
+
+func (p PublicSemantic) String() string {
+	switch p {
+	case PublicSemanticPublicEndpoint:
+		return "public_endpoint"
+	case PublicSemanticVPCEndpoint:
+		return "vpc_endpoint"
+	case PublicSemanticPublicURL:
+		return "public_url"
+	default:
+		return "none"
+	}
+}
+
+// ExposureState is the outcome of an optional safe reachability check.
+type ExposureState uint8
+
+const (
+	ExposureUnknown ExposureState = iota
+	ExposurePublicDNS
+	ExposureReachable
+	ExposureAuthRequired
+	ExposurePublicRead
+	ExposurePrivate
+	ExposureNotFound
+)
+
+func (e ExposureState) String() string {
+	switch e {
+	case ExposurePublicDNS:
+		return "public_dns"
+	case ExposureReachable:
+		return "reachable"
+	case ExposureAuthRequired:
+		return "auth_required"
+	case ExposurePublicRead:
+		return "public_read"
+	case ExposurePrivate:
+		return "private"
+	case ExposureNotFound:
+		return "not_found"
+	default:
+		return "unknown"
+	}
+}
+
+// Match is the compact result of classifying a single hostname. The string
+// fields are substrings of the input hostname (zero-copy).
+type Match struct {
+	Provider         Provider
+	Service          Service
+	PublicSemantic   PublicSemantic
+	ResourceSpecific bool
+	Region           string
+	Resource         string
+}
+
+// Candidate is a provider resource discovered directly or through DNS.
+type Candidate struct {
+	Hostname         string // hostname that matched (provider hostname)
+	Original         string // original CT hostname (equal to Hostname for direct matches)
+	Region           string
+	Resource         string
+	Provider         Provider
+	Service          Service
+	Source           Source
+	PublicSemantic   PublicSemantic
+	ResourceSpecific bool
+}

internal/pipeline/pipeline.go

@@ -0,0 +1,345 @@
+// Package pipeline wires the in-process CertSpotter identifier callback to the
+// classifier, DNS enrichment and the batched PostgreSQL writer using bounded
+// queues throughout.
+package pipeline
+
+import (
+	"context"
+	"net"
+	"sync"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+
+	"tea.karlcom.de/karl/bucketeer/internal/classify"
+	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+// Finding is a fully enriched candidate ready to be persisted.
+type Finding struct {
+	model.Candidate
+	CNAMEChain []string
+	IPv4       []string
+	IPv6       []string
+}
+
+// WriteFunc persists a batch of findings.
+type WriteFunc func(ctx context.Context, batch []Finding) error
+
+type Config struct {
+	DirectQueueSize int
+	DNSQueueSize    int
+	DBQueueSize     int
+	DirectWorkers   int
+	DNSWorkers      int
+	BatchSize       int
+	FlushInterval   time.Duration
+	DedupTTL        time.Duration
+	DedupSize       int
+
+	Enricher *dns.Enricher
+	Write    WriteFunc
+	Metrics  *metrics.Metrics
+}
+
+// Pipeline is the certificate-to-database data flow. It is safe to call
+// HandleIdentifiers from the CertSpotter monitor goroutines concurrently.
+type Pipeline struct {
+	cfg Config
+
+	directQ chan model.Candidate
+	dnsQ    chan string
+	dbQ     chan Finding
+
+	dedup    *dns.TTLCache[bool] // hostnames already queued for DNS
+	seen     *dns.TTLCache[bool] // provider+service+provider_hostname already written
+	producer sync.WaitGroup
+	workers  sync.WaitGroup
+}
+
+func New(cfg Config) *Pipeline {
+	if cfg.DirectQueueSize <= 0 {
+		cfg.DirectQueueSize = 8192
+	}
+	if cfg.DNSQueueSize <= 0 {
+		cfg.DNSQueueSize = 8192
+	}
+	if cfg.DBQueueSize <= 0 {
+		cfg.DBQueueSize = 8192
+	}
+	if cfg.DirectWorkers <= 0 {
+		cfg.DirectWorkers = 2
+	}
+	if cfg.DNSWorkers <= 0 {
+		cfg.DNSWorkers = 16
+	}
+	if cfg.BatchSize <= 0 {
+		cfg.BatchSize = 500
+	}
+	if cfg.FlushInterval <= 0 {
+		cfg.FlushInterval = 2 * time.Second
+	}
+	if cfg.DedupTTL <= 0 {
+		cfg.DedupTTL = time.Hour
+	}
+	if cfg.DedupSize <= 0 {
+		cfg.DedupSize = 1 << 20
+	}
+	p := &Pipeline{
+		cfg:     cfg,
+		directQ: make(chan model.Candidate, cfg.DirectQueueSize),
+		dnsQ:    make(chan string, cfg.DNSQueueSize),
+		dbQ:     make(chan Finding, cfg.DBQueueSize),
+		dedup:   dns.NewTTLCache[bool](cfg.DedupSize, 64),
+		seen:    dns.NewTTLCache[bool](cfg.DedupSize, 64),
+	}
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) })
+		p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(len(p.dnsQ)) })
+		p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) })
+	}
+	return p
+}
+
+// Start launches the worker pools. Close must be called to drain.
+func (p *Pipeline) Start(ctx context.Context) {
+	for i := 0; i < p.cfg.DirectWorkers; i++ {
+		p.producer.Add(1)
+		go p.directWorker(ctx)
+	}
+	for i := 0; i < p.cfg.DNSWorkers; i++ {
+		p.producer.Add(1)
+		go p.dnsWorker(ctx)
+	}
+	p.workers.Add(1)
+	go p.dbWorker(ctx)
+}
+
+// Close stops intake and drains all queues. It must only be called after the
+// CertSpotter monitor has stopped calling HandleIdentifiers.
+func (p *Pipeline) Close() {
+	close(p.directQ)
+	close(p.dnsQ)
+	p.producer.Wait()
+	close(p.dbQ)
+	p.workers.Wait()
+}
+
+// HandleIdentifiers implements certspotter.IdentifierSink. It is deliberately
+// allocation-light and never blocks: queues are bounded and overflow is dropped.
+func (p *Pipeline) HandleIdentifiers(ctx context.Context, ids *certspotter.Identifiers) {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.CTCertificates.Add(1)
+		p.cfg.Metrics.CTIdentifiers.Add(uint64(len(ids.DNSNames)))
+	}
+	for _, host := range ids.DNSNames {
+		p.HandleHost(host)
+	}
+}
+
+// HandleHost classifies a single hostname and enqueues it for direct handling
+// or, when it does not match, for DNS/CNAME enrichment.
+func (p *Pipeline) HandleHost(host string) {
+	if m, ok := classify.Classify(host); ok {
+		if !m.ResourceSpecific {
+			// Generic service endpoints (bucket name in the path) are not
+			// customer resources; do not create findings for them.
+			return
+		}
+		if p.cfg.Metrics != nil {
+			p.cfg.Metrics.DirectMatches.Add(1)
+			p.cfg.Metrics.Candidates.Add(1)
+		}
+		if !p.markSeen(host) {
+			return
+		}
+		cand := candidateFrom(host, host, m, model.SourceDirectCT)
+		select {
+		case p.directQ <- cand:
+		default:
+			p.drop()
+		}
+		return
+	}
+	if !classify.ShouldEnrich(host) {
+		return
+	}
+	if _, dup := p.dedup.Get(host); dup {
+		return
+	}
+	p.dedup.Set(host, true, p.cfg.DedupTTL)
+	select {
+	case p.dnsQ <- host:
+	default:
+		p.drop()
+	}
+}
+
+func (p *Pipeline) drop() {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.QueueFull.Add(1)
+		p.cfg.Metrics.QueueDrops.Add(1)
+	}
+}
+
+func candidateFrom(host, original string, m model.Match, src model.Source) model.Candidate {
+	return model.Candidate{
+		Hostname:         host,
+		Original:         original,
+		Region:           m.Region,
+		Resource:         m.Resource,
+		Provider:         m.Provider,
+		Service:          m.Service,
+		Source:           src,
+		PublicSemantic:   m.PublicSemantic,
+		ResourceSpecific: m.ResourceSpecific,
+	}
+}
+
+func appendIPs(f *Finding, ips []net.IP) {
+	for _, ip := range ips {
+		if v4 := ip.To4(); v4 != nil {
+			f.IPv4 = append(f.IPv4, v4.String())
+		} else {
+			f.IPv6 = append(f.IPv6, ip.String())
+		}
+	}
+}
+
+func (p *Pipeline) enrich(ctx context.Context, c model.Candidate) Finding {
+	f := Finding{Candidate: c}
+	if p.cfg.Enricher == nil {
+		return f
+	}
+	res, ok := p.cfg.Enricher.Resolve(ctx, c.Hostname)
+	if !ok {
+		return f
+	}
+	f.CNAMEChain = res.Chain
+	appendIPs(&f, res.IPs)
+	return f
+}
+
+func (p *Pipeline) directWorker(ctx context.Context) {
+	defer p.producer.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case c, ok := <-p.directQ:
+			if !ok {
+				return
+			}
+			p.pushDB(ctx, p.enrich(ctx, c))
+		}
+	}
+}
+
+func (p *Pipeline) dnsWorker(ctx context.Context) {
+	defer p.producer.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case host, ok := <-p.dnsQ:
+			if !ok {
+				return
+			}
+			if len(p.dbQ) >= cap(p.dbQ)/2 {
+				// Load shedding: optional CNAME enrichment is dropped before
+				// direct matches when the database writer is backed up.
+				p.drop()
+				continue
+			}
+			p.enrichHost(ctx, host)
+		}
+	}
+}
+
+func (p *Pipeline) enrichHost(ctx context.Context, host string) {
+	res, ok := p.cfg.Enricher.Resolve(ctx, host)
+	if !ok {
+		return
+	}
+	for _, target := range res.Chain[1:] {
+		m, matched := classify.Classify(target)
+		if !matched || !m.ResourceSpecific {
+			continue
+		}
+		if p.cfg.Metrics != nil {
+			p.cfg.Metrics.CNAMEMatches.Add(1)
+			p.cfg.Metrics.Candidates.Add(1)
+		}
+		if !p.markSeen(target) {
+			continue
+		}
+		f := Finding{
+			Candidate:  candidateFrom(target, host, m, model.SourceCNAME),
+			CNAMEChain: res.Chain,
+		}
+		appendIPs(&f, res.IPs)
+		p.pushDB(ctx, f)
+	}
+}
+
+// markSeen dedups by hostname alone: a given hostname always classifies to the
+// same provider/service, so no key concatenation (and no allocation) is needed.
+func (p *Pipeline) markSeen(host string) bool {
+	if _, dup := p.seen.Get(host); dup {
+		return false
+	}
+	p.seen.Set(host, true, p.cfg.DedupTTL)
+	return true
+}
+
+func (p *Pipeline) pushDB(ctx context.Context, f Finding) {
+	select {
+	case p.dbQ <- f:
+	default:
+		p.drop()
+	}
+}
+
+func (p *Pipeline) dbWorker(ctx context.Context) {
+	defer p.workers.Done()
+	batch := make([]Finding, 0, p.cfg.BatchSize)
+	ticker := time.NewTicker(p.cfg.FlushInterval)
+	defer ticker.Stop()
+
+	flush := func() {
+		if len(batch) == 0 {
+			return
+		}
+		if p.cfg.Write != nil {
+			if err := p.cfg.Write(ctx, batch); err != nil {
+				if p.cfg.Metrics != nil {
+					p.cfg.Metrics.DBErrors.Add(1)
+				}
+			} else if p.cfg.Metrics != nil {
+				p.cfg.Metrics.DBRows.Add(uint64(len(batch)))
+			}
+		}
+		batch = batch[:0]
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			flush()
+			return
+		case f, ok := <-p.dbQ:
+			if !ok {
+				flush()
+				return
+			}
+			batch = append(batch, f)
+			if len(batch) >= p.cfg.BatchSize {
+				flush()
+			}
+		case <-ticker.C:
+			flush()
+		}
+	}
+}

internal/pipeline/pipeline_test.go

@@ -0,0 +1,236 @@
+package pipeline
+
+import (
+	"context"
+	"net"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+
+	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func makeHosts(prefix, suffix string) []string {
+	hosts := make([]string, 256)
+	for i := range hosts {
+		hosts[i] = prefix + strconv.Itoa(i) + suffix
+	}
+	return hosts
+}
+
+// TestHandleHostNoAllocs audits the sink hot path: after warmup, classifying and
+// enqueueing a hostname must not allocate.
+func TestHandleHostNoAllocs(t *testing.T) {
+	p := New(Config{DNSQueueSize: 1 << 20, DedupSize: 1 << 20})
+	nonMatch := makeHosts("api", ".example.com")
+	direct := makeHosts("bucket", ".s3.eu-central-1.amazonaws.com")
+	for _, h := range nonMatch {
+		p.HandleHost(h)
+	}
+	for _, h := range direct {
+		p.HandleHost(h)
+	}
+	allocs := testing.AllocsPerRun(2000, func() {
+		for i := 0; i < 64; i++ {
+			p.HandleHost(nonMatch[i])
+			p.HandleHost(direct[i])
+		}
+	})
+	if allocs != 0 {
+		t.Fatalf("HandleHost allocates %.3f allocs/op, want 0", allocs)
+	}
+}
+
+func BenchmarkHandleHostNonMatch(b *testing.B) {
+	p := New(Config{DNSQueueSize: 1 << 20, DedupSize: 1 << 20})
+	hosts := makeHosts("api", ".example.com")
+	for _, h := range hosts {
+		p.HandleHost(h)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.HandleHost(hosts[i&255])
+	}
+}
+
+func BenchmarkHandleHostDirectMatch(b *testing.B) {
+	p := New(Config{DirectQueueSize: 1 << 20, DedupSize: 1 << 20})
+	hosts := makeHosts("bucket", ".s3.eu-central-1.amazonaws.com")
+	for _, h := range hosts {
+		p.HandleHost(h)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.HandleHost(hosts[i&255])
+	}
+}
+
+type fakeResolver struct {
+	cnames map[string]string
+	ips    map[string][]net.IP
+}
+
+func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	if c, ok := f.cnames[host]; ok {
+		return c, nil
+	}
+	return host, nil
+}
+
+func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	return f.ips[host], nil
+}
+
+func newEnricher(r dns.Resolver) *dns.Enricher {
+	return &dns.Enricher{
+		Resolver:    r,
+		Cache:       dns.NewTTLCache[dns.Result](1024, 8),
+		PositiveTTL: time.Minute,
+		NegativeTTL: time.Minute,
+		MaxDepth:    8,
+	}
+}
+
+type collector struct {
+	mu    sync.Mutex
+	items []Finding
+}
+
+func (c *collector) write(ctx context.Context, batch []Finding) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.items = append(c.items, batch...)
+	return nil
+}
+
+func (c *collector) all() []Finding {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return append([]Finding(nil), c.items...)
+}
+
+func TestDirectMatch(t *testing.T) {
+	col := &collector{}
+	m := metrics.New()
+	p := New(Config{
+		Enricher:  newEnricher(&fakeResolver{}),
+		Write:     col.write,
+		Metrics:   m,
+		BatchSize: 10,
+	})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{"bucket.s3.eu-central-1.amazonaws.com"}})
+	time.Sleep(50 * time.Millisecond)
+	p.Close()
+
+	items := col.all()
+	if len(items) != 1 {
+		t.Fatalf("got %d findings, want 1 (%+v)", len(items), items)
+	}
+	f := items[0]
+	if f.Provider != model.ProviderAWS || f.Service != model.ServiceS3 || f.Source != model.SourceDirectCT {
+		t.Fatalf("unexpected finding %+v", f)
+	}
+	if m.DirectMatches.Load() != 1 {
+		t.Fatalf("direct matches = %d, want 1", m.DirectMatches.Load())
+	}
+}
+
+func TestCNAMEMatch(t *testing.T) {
+	col := &collector{}
+	r := &fakeResolver{
+		cnames: map[string]string{"files.example.com": "files.s3.amazonaws.com"},
+		ips:    map[string][]net.IP{"files.example.com": {net.ParseIP("1.2.3.4")}},
+	}
+	m := metrics.New()
+	p := New(Config{Enricher: newEnricher(r), Write: col.write, Metrics: m, BatchSize: 10})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{"files.example.com"}})
+	time.Sleep(50 * time.Millisecond)
+	p.Close()
+
+	items := col.all()
+	if len(items) != 1 {
+		t.Fatalf("got %d findings, want 1", len(items))
+	}
+	f := items[0]
+	if f.Source != model.SourceCNAME || f.Hostname != "files.s3.amazonaws.com" || f.Original != "files.example.com" {
+		t.Fatalf("unexpected finding %+v", f)
+	}
+	if len(f.IPv4) != 1 || f.IPv4[0] != "1.2.3.4" {
+		t.Fatalf("unexpected IPs %+v", f.IPv4)
+	}
+	if m.CNAMEMatches.Load() != 1 {
+		t.Fatalf("cname matches = %d, want 1", m.CNAMEMatches.Load())
+	}
+}
+
+func TestCNAMEMatchIPv6(t *testing.T) {
+	col := &collector{}
+	r := &fakeResolver{
+		cnames: map[string]string{"db.example.com": "mydb.cluster-abc.us-east-1.rds.amazonaws.com"},
+		ips:    map[string][]net.IP{"db.example.com": {net.ParseIP("2606:4700::1111")}},
+	}
+	p := New(Config{Enricher: newEnricher(r), Write: col.write, BatchSize: 10})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleHost("db.example.com")
+	time.Sleep(50 * time.Millisecond)
+	p.Close()
+	items := col.all()
+	if len(items) != 1 || len(items[0].IPv6) != 1 || items[0].IPv6[0] != "2606:4700::1111" {
+		t.Fatalf("unexpected IPv6 finding: %+v", items)
+	}
+}
+
+func TestQueueSaturationDrops(t *testing.T) {
+	m := metrics.New()
+	p := New(Config{DirectQueueSize: 1, DNSQueueSize: 1, Metrics: m})
+	// Intentionally do not Start: the queue stays full.
+	p.HandleHost("bucket.s3.amazonaws.com")
+	p.HandleHost("other.s3.amazonaws.com")
+	if m.QueueDrops.Load() == 0 {
+		t.Fatal("expected a drop when the direct queue is full")
+	}
+	if m.Candidates.Load() != 2 {
+		t.Fatalf("candidates = %d, want 2", m.Candidates.Load())
+	}
+}
+
+func TestDNSDedup(t *testing.T) {
+	m := metrics.New()
+	p := New(Config{DNSQueueSize: 8, Metrics: m})
+	p.HandleHost("files.example.com")
+	p.HandleHost("files.example.com")
+	if got := len(p.dnsQ); got != 1 {
+		t.Fatalf("dns queue len = %d, want 1", got)
+	}
+}
+
+func TestGenericEndpointNotStored(t *testing.T) {
+	col := &collector{}
+	p := New(Config{Write: col.write, BatchSize: 10})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{
+		"objectstorage.us-ashburn-1.oraclecloud.com",
+	}})
+	time.Sleep(30 * time.Millisecond)
+	p.Close()
+	if items := col.all(); len(items) != 0 {
+		t.Fatalf("generic endpoint produced findings: %+v", items)
+	}
+}

internal/probe/probe.go

@@ -0,0 +1,142 @@
+// Package probe performs optional, minimal, non-destructive reachability checks.
+// It never authenticates, downloads objects, mutates anything, or runs database
+// queries; it only observes the response to a single HTTPS GET /.
+package probe
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+// UpdateFunc records the observed exposure state.
+type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState)
+
+type Config struct {
+	Workers int
+	Timeout time.Duration
+	Queue   int
+	Metrics *metrics.Metrics
+	Update  UpdateFunc
+}
+
+// Prober runs a fixed pool of workers over a bounded queue.
+type Prober struct {
+	cfg    Config
+	queue  chan model.Candidate
+	client *http.Client
+	wg     sync.WaitGroup
+}
+
+func New(cfg Config) *Prober {
+	if cfg.Workers <= 0 {
+		cfg.Workers = 4
+	}
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = 5 * time.Second
+	}
+	if cfg.Queue <= 0 {
+		cfg.Queue = 1024
+	}
+	return &Prober{
+		cfg:   cfg,
+		queue: make(chan model.Candidate, cfg.Queue),
+		client: &http.Client{
+			Timeout: cfg.Timeout,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+			Transport: &http.Transport{
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true}, // observation only, no trust decision
+				MaxIdleConns:        cfg.Workers * 2,
+				MaxIdleConnsPerHost: 2,
+				IdleConnTimeout:     30 * time.Second,
+				DialContext: (&net.Dialer{
+					Timeout:   cfg.Timeout,
+					KeepAlive: 30 * time.Second,
+				}).DialContext,
+			},
+		},
+	}
+}
+
+func (p *Prober) Start(ctx context.Context) {
+	for i := 0; i < p.cfg.Workers; i++ {
+		p.wg.Add(1)
+		go p.worker(ctx)
+	}
+}
+
+// Submit enqueues a candidate without blocking; it drops when the queue is full.
+func (p *Prober) Submit(c model.Candidate) {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.ProbeAttempts.Add(1)
+	}
+	select {
+	case p.queue <- c:
+	default:
+	}
+}
+
+func (p *Prober) Wait() { p.wg.Wait() }
+
+func (p *Prober) worker(ctx context.Context) {
+	defer p.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case c, ok := <-p.queue:
+			if !ok {
+				return
+			}
+			state := p.probe(ctx, c)
+			if p.cfg.Metrics != nil && state == model.ExposurePublicRead {
+				p.cfg.Metrics.ProbePublic.Add(1)
+			}
+			if p.cfg.Update != nil {
+				p.cfg.Update(ctx, c, state)
+			}
+		}
+	}
+}
+
+func (p *Prober) probe(ctx context.Context, c model.Candidate) model.ExposureState {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+c.Hostname+"/", nil)
+	if err != nil {
+		return model.ExposureUnknown
+	}
+	resp, err := p.client.Do(req)
+	if err != nil {
+		var dnsErr *net.DNSError
+		if errors.As(err, &dnsErr) {
+			return model.ExposurePrivate
+		}
+		return model.ExposureUnknown
+	}
+	defer resp.Body.Close()
+
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		return model.ExposurePublicRead
+	case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized:
+		return model.ExposureAuthRequired
+	case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone:
+		return model.ExposureNotFound
+	default:
+		return model.ExposureReachable
+	}
+}
+
+// Describe is a helper for logs/tests.
+func Describe(s model.ExposureState) string {
+	return fmt.Sprintf("exposure=%s", s)
+}

internal/probe/probe_test.go

@@ -0,0 +1,45 @@
+package probe
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func newProber(t *testing.T, status int) (*Prober, string) {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(status)
+	}))
+	t.Cleanup(srv.Close)
+	p := New(Config{Workers: 1, Timeout: time.Second})
+	// httptest uses a self-signed cert; the prober already skips verification.
+	_ = tls.Config{}
+	host := strings.TrimPrefix(srv.URL, "https://")
+	return p, host
+}
+
+func TestProbeClassification(t *testing.T) {
+	cases := []struct {
+		status int
+		want   model.ExposureState
+	}{
+		{http.StatusOK, model.ExposurePublicRead},
+		{http.StatusForbidden, model.ExposureAuthRequired},
+		{http.StatusUnauthorized, model.ExposureAuthRequired},
+		{http.StatusNotFound, model.ExposureNotFound},
+		{http.StatusInternalServerError, model.ExposureReachable},
+	}
+	for _, c := range cases {
+		p, host := newProber(t, c.status)
+		got := p.probe(t.Context(), model.Candidate{Hostname: host})
+		if got != c.want {
+			t.Errorf("status %d: got %s, want %s", c.status, got, c.want)
+		}
+	}
+}

internal/store/state.go

@@ -0,0 +1,165 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"strconv"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/monitor"
+)
+
+var _ monitor.StateProvider = (*Store)(nil)
+
+// Prepare implements monitor.StateProvider. It applies the embedded schema so
+// the container needs no writable filesystem.
+func (s *Store) Prepare(ctx context.Context) error {
+	if err := s.Migrate(ctx); err != nil {
+		return err
+	}
+	return nil
+}
+
+// PrepareLog is a no-op: checkpoint rows are created lazily by StoreLogState.
+func (s *Store) PrepareLog(ctx context.Context, logID monitor.LogID) error { return nil }
+
+// StoreLogState persists the certspotter monitor position for a log.
+func (s *Store) StoreLogState(ctx context.Context, logID monitor.LogID, state *monitor.LogState) error {
+	data, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+	_, err = s.pool.Exec(ctx, `
+		INSERT INTO ct_checkpoints (log_id, state, updated_at)
+		VALUES ($1, $2, now())
+		ON CONFLICT (log_id) DO UPDATE SET state = EXCLUDED.state, updated_at = now()`,
+		logID.Base64URLString(), data)
+	return err
+}
+
+// LoadLogState returns nil when no checkpoint exists for the log.
+func (s *Store) LoadLogState(ctx context.Context, logID monitor.LogID) (*monitor.LogState, error) {
+	var data []byte
+	err := s.pool.QueryRow(ctx, `SELECT state FROM ct_checkpoints WHERE log_id = $1`, logID.Base64URLString()).Scan(&data)
+	if err == pgx.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	state := new(monitor.LogState)
+	if err := json.Unmarshal(data, state); err != nil {
+		return nil, fmt.Errorf("corrupt checkpoint for log %s: %w", logID.Base64URLString(), err)
+	}
+	return state, nil
+}
+
+func sthKey(sth *cttypes.SignedTreeHead) string {
+	return strconv.FormatUint(sth.TreeSize, 10) + "-" +
+		strconv.FormatUint(sth.Timestamp, 10) + "-" +
+		sth.RootHash.Base64String()
+}
+
+// StoreSTH stores an unverified signed tree head. If it already exists the
+// original stored_at is preserved.
+func (s *Store) StoreSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) (*monitor.StoredSTH, error) {
+	data, err := json.Marshal(sth)
+	if err != nil {
+		return nil, err
+	}
+	var storedAt time.Time
+	err = s.pool.QueryRow(ctx, `
+		INSERT INTO ct_sths (log_id, sth_key, tree_size, sth, stored_at)
+		VALUES ($1, $2, $3, $4, now())
+		ON CONFLICT (log_id, sth_key) DO UPDATE SET log_id = EXCLUDED.log_id
+		RETURNING stored_at`,
+		logID.Base64URLString(), sthKey(sth), sth.TreeSize, data).Scan(&storedAt)
+	if err != nil {
+		return nil, err
+	}
+	return &monitor.StoredSTH{SignedTreeHead: *sth, StoredAt: storedAt}, nil
+}
+
+// LoadSTHs loads all stored STHs for a log, sorted by tree size.
+func (s *Store) LoadSTHs(ctx context.Context, logID monitor.LogID) ([]*monitor.StoredSTH, error) {
+	rows, err := s.pool.Query(ctx,
+		`SELECT sth, stored_at FROM ct_sths WHERE log_id = $1 ORDER BY tree_size ASC`,
+		logID.Base64URLString())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []*monitor.StoredSTH
+	for rows.Next() {
+		var (
+			data     []byte
+			storedAt time.Time
+		)
+		if err := rows.Scan(&data, &storedAt); err != nil {
+			return nil, err
+		}
+		var sth cttypes.SignedTreeHead
+		if err := json.Unmarshal(data, &sth); err != nil {
+			return nil, err
+		}
+		out = append(out, &monitor.StoredSTH{SignedTreeHead: sth, StoredAt: storedAt})
+	}
+	return out, rows.Err()
+}
+
+// RemoveSTH deletes an STH once it has been verified against the Merkle tree.
+func (s *Store) RemoveSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) error {
+	_, err := s.pool.Exec(ctx, `DELETE FROM ct_sths WHERE log_id = $1 AND sth_key = $2`,
+		logID.Base64URLString(), sthKey(sth))
+	return err
+}
+
+// StoreIssuer and LoadIssuer are no-ops. ctcloud's identifier sink bypasses
+// certificate-chain retrieval entirely, so the issuer cache is never consulted;
+// keeping it empty also satisfies the "no certificate contents in the state
+// store" requirement. The Go runtime cache handles any incidental calls.
+func (s *Store) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error {
+	return nil
+}
+
+func (s *Store) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	return nil, nil
+}
+
+// NotifyCert is never called because the identifier sink handles certificates.
+func (s *Store) NotifyCert(ctx context.Context, cert *monitor.DiscoveredCert) error { return nil }
+
+// NotifyMalformedEntry counts unparseable log entries without logging per entry.
+func (s *Store) NotifyMalformedEntry(ctx context.Context, entry *monitor.LogEntry, parseError error) error {
+	if s.metrics != nil {
+		s.metrics.MalformedCT.Add(1)
+	}
+	return nil
+}
+
+// NotifyHealthCheckFailure is rare (once per health-check interval at most), so
+// it is safe to log.
+func (s *Store) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info monitor.HealthCheckFailure) error {
+	log.Printf("ct health check failure: %s", info.Summary())
+	return nil
+}
+
+// NotifyError counts transient CT errors. It deliberately does not log, to keep
+// logging off the high-frequency error path.
+func (s *Store) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error {
+	if s.metrics != nil {
+		s.metrics.CTErrors.Add(1)
+	}
+	return nil
+}
+
+// GetErrors returns the recent CT errors. ctcloud keeps only aggregate counters.
+func (s *Store) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
+	return "", nil
+}

internal/store/store.go

@@ -0,0 +1,146 @@
+// Package store owns the PostgreSQL schema, the batched findings writer and the
+// restart-safe CT checkpoint state provider.
+package store
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
+	"tea.karlcom.de/karl/bucketeer/migrations"
+)
+
+// Store wraps a pgx pool with the operations ctcloud needs.
+type Store struct {
+	pool    *pgxpool.Pool
+	metrics *metrics.Metrics
+}
+
+// Open connects to PostgreSQL. The pool is bounded; the caller configures the
+// maximum number of connections.
+func Open(ctx context.Context, databaseURL string, maxConns int) (*Store, error) {
+	cfg, err := pgxpool.ParseConfig(databaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid DATABASE_URL: %w", err)
+	}
+	if maxConns > 0 {
+		cfg.MaxConns = int32(maxConns)
+	}
+	cfg.MaxConnLifetime = 30 * time.Minute
+	cfg.MinConns = 1
+	pool, err := pgxpool.NewWithConfig(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &Store{pool: pool}, nil
+}
+
+func (s *Store) SetMetrics(m *metrics.Metrics) { s.metrics = m }
+
+func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
+
+func (s *Store) Close() { s.pool.Close() }
+
+func (s *Store) Pool() *pgxpool.Pool { return s.pool }
+
+// Migrate applies the embedded schema idempotently.
+func (s *Store) Migrate(ctx context.Context) error {
+	for _, name := range migrations.SQL() {
+		sqlBytes, err := migrations.FS.ReadFile(name)
+		if err != nil {
+			return err
+		}
+		if _, err := s.pool.Exec(ctx, string(sqlBytes)); err != nil {
+			return fmt.Errorf("applying %s: %w", name, err)
+		}
+	}
+	return nil
+}
+
+const upsertFindingSQL = `
+WITH asset AS (
+    INSERT INTO cloud_assets
+        (provider, service, hostname, provider_hostname, resource_identifier,
+         region, resource_specific, public_semantic, first_seen, last_seen)
+    VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now())
+    ON CONFLICT (provider, service, provider_hostname)
+    DO UPDATE SET
+        last_seen           = now(),
+        hostname            = EXCLUDED.hostname,
+        resource_identifier = CASE WHEN EXCLUDED.resource_identifier = '' THEN cloud_assets.resource_identifier ELSE EXCLUDED.resource_identifier END,
+        region              = CASE WHEN EXCLUDED.region = '' THEN cloud_assets.region ELSE EXCLUDED.region END
+    RETURNING id
+)
+INSERT INTO observations
+    (asset_id, source_hostname, source_type, cname_chain, ipv4, ipv6)
+SELECT asset.id, $9, $10, $11, $12, $13 FROM asset`
+
+// WriteFindings writes one batch of findings in a single round trip. Asset rows
+// are upserted and an observation is appended for each finding.
+func (s *Store) WriteFindings(ctx context.Context, batch []pipeline.Finding) error {
+	if len(batch) == 0 {
+		return nil
+	}
+	var lastErr error
+	// ponytail: three quick attempts; the pipeline keeps running and the
+	// bounded DB queue sheds load if PostgreSQL stays down.
+	for attempt := 0; attempt < 3; attempt++ {
+		if err := s.writeBatch(ctx, batch); err != nil {
+			lastErr = err
+			if ctx.Err() != nil {
+				return err
+			}
+			backoff := time.Duration(1<<attempt) * 100 * time.Millisecond
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(backoff):
+			}
+			continue
+		}
+		return nil
+	}
+	return lastErr
+}
+
+func (s *Store) writeBatch(ctx context.Context, batch []pipeline.Finding) error {
+	b := &pgx.Batch{}
+	for i := range batch {
+		f := &batch[i]
+		b.Queue(upsertFindingSQL,
+			f.Provider.String(), f.Service.String(), f.Original, f.Hostname,
+			f.Resource, f.Region, f.ResourceSpecific, f.PublicSemantic.String(),
+			f.Original, f.Source.String(),
+			nonNil(f.CNAMEChain), nonNil(f.IPv4), nonNil(f.IPv6),
+		)
+	}
+	br := s.pool.SendBatch(ctx, b)
+	defer br.Close()
+	for range batch {
+		if _, err := br.Exec(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func nonNil(s []string) []string {
+	if s == nil {
+		return []string{}
+	}
+	return s
+}
+
+// UpdateExposure records the outcome of an optional reachability probe.
+func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState) error {
+	_, err := s.pool.Exec(ctx,
+		`UPDATE cloud_assets SET exposure_state = $4 WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
+		provider, service, providerHostname, state.String())
+	return err
+}

internal/store/store_test.go

@@ -0,0 +1,66 @@
+package store
+
+import (
+	"os"
+	"testing"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
+)
+
+func TestSTHKeyStableAndUnique(t *testing.T) {
+	a := &cttypes.SignedTreeHead{TreeSize: 10, Timestamp: 100}
+	if sthKey(a) != sthKey(a) {
+		t.Fatal("sthKey is not deterministic")
+	}
+	b := &cttypes.SignedTreeHead{TreeSize: 11, Timestamp: 100}
+	if sthKey(a) == sthKey(b) {
+		t.Fatal("sthKey collides for different tree sizes")
+	}
+}
+
+func TestNonNil(t *testing.T) {
+	if got := nonNil(nil); got == nil || len(got) != 0 {
+		t.Fatalf("nonNil(nil) = %#v", got)
+	}
+	in := []string{"a"}
+	if got := nonNil(in); len(got) != 1 || got[0] != "a" {
+		t.Fatalf("nonNil changed slice: %#v", got)
+	}
+}
+
+// TestIntegration exercises the real schema when DATABASE_URL is set.
+func TestIntegration(t *testing.T) {
+	url := os.Getenv("DATABASE_URL")
+	if url == "" {
+		t.Skip("DATABASE_URL not set; skipping integration test")
+	}
+	ctx := t.Context()
+	s, err := Open(ctx, url, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	if err := s.Migrate(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	f := pipeline.Finding{
+		Candidate: model.Candidate{
+			Hostname: "it-bucket.s3.eu-central-1.amazonaws.com",
+			Original: "it-bucket.s3.eu-central-1.amazonaws.com",
+			Provider: model.ProviderAWS,
+			Service:  model.ServiceS3,
+			Source:   model.SourceDirectCT,
+		},
+		IPv4: []string{"1.2.3.4"},
+	}
+	if err := s.WriteFindings(ctx, []pipeline.Finding{f}); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.UpdateExposure(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, model.ExposureReachable); err != nil {
+		t.Fatal(err)
+	}
+}