ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

internal/classify/aws.go

@@ -0,0 +1,148 @@
+package classify
+
+import (
+	"strings"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match {
+	return model.Match{
+		Provider:         p,
+		Service:          s,
+		PublicSemantic:   sem,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+func classifyAWS(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {
+		return model.Match{}, false
+	}
+	if m, ok := classifyAWSS3(s); ok {
+		return m, true
+	}
+
+	// Service-after-region forms: <resource>.<region>.<service>.amazonaws.com
+	if n >= 5 {
+		switch s.label(n - 3) {
+		case "rds":
+			return awsResourceAfterRegion(s, model.ServiceRDS)
+		case "docdb":
+			return awsResourceAfterRegion(s, model.ServiceDocumentDB)
+		case "neptune":
+			return awsResourceAfterRegion(s, model.ServiceNeptune)
+		case "redshift", "redshift-serverless":
+			return awsResourceAfterRegion(s, model.ServiceRedshift)
+		case "es":
+			first := s.label(0)
+			switch {
+			case strings.HasPrefix(first, "search-"):
+				return aws(model.ProviderAWS, model.ServiceOpenSearch,
+					model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true
+			case strings.HasPrefix(first, "vpc-"):
+				return aws(model.ProviderAWS, model.ServiceOpenSearch,
+					model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true
+			}
+		case "cache":
+			return aws(model.ProviderAWS, model.ServiceElastiCache,
+				model.PublicSemanticNone, s.label(0), s.label(n-4)), true
+		}
+	}
+
+	// Service-before-region form: <resource>.memorydb.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "memorydb" {
+		return aws(model.ProviderAWS, model.ServiceMemoryDB,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+	return model.Match{}, false
+}
+
+// awsResourceAfterRegion matches <resource>.<region>.<service>.amazonaws.com
+// and rejects the bare <service>.<region>.amazonaws.com service endpoint.
+func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) {
+	n := s.n
+	if n < 5 {
+		return model.Match{}, false
+	}
+	resource := s.label(0)
+	if resource == s.label(n-3) {
+		return model.Match{}, false
+	}
+	return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true
+}
+
+func classifyAWSS3(s *scanner) (model.Match, bool) {
+	n := s.n
+
+	// Access points: <ap>.s3-accesspoint[-fips][.dualstack].<region>.amazonaws.com
+	if n >= 5 {
+		svcIdx := -1
+		switch {
+		case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips":
+			svcIdx = n - 4
+		case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint":
+			svcIdx = n - 5
+		}
+		if svcIdx > 0 {
+			res := s.label(0)
+			if res != "s3-accesspoint" && res != "s3-accesspoint-fips" {
+				return aws(model.ProviderAWS, model.ServiceS3,
+					model.PublicSemanticNone, res, s.label(n-3)), true
+			}
+		}
+	}
+
+	// Multi-region access point: <res>.mrap.accesspoint.s3-global.amazonaws.com
+	if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" &&
+		s.label(n-3) == "s3-global" {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), ""), true
+	}
+
+	// S3 Express: <bucket>.s3express-<zone>.<region>.amazonaws.com
+	if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+
+	// S3 website: <bucket>.s3-website.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "s3-website" {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+	// S3 website (region in the service label): <bucket>.s3-website-<region>.amazonaws.com
+	if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), ""), true
+	}
+
+	// Plain S3 without region: <bucket>.s3.amazonaws.com
+	if n == 4 && s.label(n-3) == "s3" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true
+	}
+	// Plain S3 with region: <bucket>.s3.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "s3" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
+	}
+	// Dualstack S3: <bucket>.s3.dualstack.<region>.amazonaws.com
+	if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
+	}
+	return model.Match{}, false
+}

internal/classify/azure.go

@@ -0,0 +1,113 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+func azure(service model.Service, resource, region string) model.Match {
+	return model.Match{
+		Provider:         model.ProviderAzure,
+		Service:          service,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+// classifyAzureCore handles *.blob|dfs|file|queue|table|web.core.windows.net.
+func classifyAzureCore(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "core" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 4)
+	resource := s.label(0)
+	if resource == "" {
+		return model.Match{}, false
+	}
+	switch svc {
+	case "blob", "dfs":
+		return azure(model.ServiceBlobStorage, resource, ""), true
+	case "file":
+		return azure(model.ServiceFileStorage, resource, ""), true
+	case "queue":
+		return azure(model.ServiceQueue, resource, ""), true
+	case "table":
+		return azure(model.ServiceTableStorage, resource, ""), true
+	case "web":
+		return azure(model.ServiceWebApp, resource, ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureDatabase handles <server>.database.windows.net.
+func classifyAzureDatabase(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "database" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceDatabase, s.label(0), ""), true
+}
+
+// classifyAzureFlexibleServer handles <server>.{postgres,mysql}.database.azure.com.
+func classifyAzureFlexibleServer(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "database" {
+		return model.Match{}, false
+	}
+	switch s.label(n - 4) {
+	case "postgres":
+		return azure(model.ServicePostgres, s.label(0), ""), true
+	case "mysql":
+		return azure(model.ServiceMySQL, s.label(0), ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureDocuments handles <account>.documents.azure.com (Cosmos DB).
+func classifyAzureDocuments(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "documents" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceCosmosDB, s.label(0), ""), true
+}
+
+// classifyAzureCosmos handles <account>.{mongo,cassandra,gremlin}.cosmos.azure.com.
+func classifyAzureCosmos(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "cosmos" {
+		return model.Match{}, false
+	}
+	switch s.label(n - 4) {
+	case "mongo", "cassandra", "gremlin":
+		return azure(model.ServiceCosmosDB, s.label(0), ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureRedis handles <name>.redis.cache.windows.net.
+func classifyAzureRedis(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" ||
+		s.label(n-3) != "cache" || s.label(n-4) != "redis" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceRedis, s.label(0), ""), true
+}
+
+// classifyAzureManagedRedis handles <name>.redis.azure.net.
+func classifyAzureManagedRedis(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "azure" || s.label(n-3) != "redis" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceRedis, s.label(0), ""), true
+}
+
+// classifyAzureSearch handles <service>.search.windows.net.
+func classifyAzureSearch(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "search" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceSearch, s.label(0), ""), true
+}

internal/classify/classify.go

@@ -0,0 +1,198 @@
+// Package classify maps a DNS hostname to a cloud/database/storage resource.
+//
+// The hot path is deliberately allocation-free: it never lowercases unless an
+// uppercase byte is present, never splits the hostname into a []string, never
+// uses regexp/JSON/reflection, and returns substrings of the input as the
+// region/resource. See BenchmarkClassify* for the 0 allocs/op assertion.
+package classify
+
+import (
+	"strings"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+const maxLabels = 64
+
+// scanner holds the byte offsets of each dot-separated label of a hostname.
+// It is stack allocated by Classify and never heap allocated.
+type scanner struct {
+	host  string
+	n     int
+	start [maxLabels]int32
+}
+
+func newScanner(host string) scanner {
+	var s scanner
+	s.host = host
+	start := int32(0)
+	for i := 0; i < len(host); i++ {
+		if host[i] == '.' {
+			if s.n >= maxLabels {
+				s.n = -1
+				return s
+			}
+			s.start[s.n] = start
+			s.n++
+			start = int32(i + 1)
+		}
+	}
+	if s.n >= maxLabels {
+		s.n = -1
+		return s
+	}
+	s.start[s.n] = start
+	s.n++
+	return s
+}
+
+// label returns the i-th label (0 = leftmost). Callers must ensure i < n.
+func (s *scanner) label(i int) string {
+	end := int32(len(s.host))
+	if i+1 < s.n {
+		end = s.start[i+1] - 1
+	}
+	return s.host[s.start[i]:end]
+}
+
+func hasUpper(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= 'A' && s[i] <= 'Z' {
+			return true
+		}
+	}
+	return false
+}
+
+func trimTrailingDots(s string) string {
+	for len(s) > 0 && s[len(s)-1] == '.' {
+		s = s[:len(s)-1]
+	}
+	return s
+}
+
+func validHost(host string) bool {
+	if host == "" || len(host) > 253 {
+		return false
+	}
+	for i := 0; i < len(host); i++ {
+		c := host[i]
+		switch {
+		case c >= 'a' && c <= 'z':
+		case c >= '0' && c <= '9':
+		case c == '-' || c == '.' || c == '_':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+// Classify returns the provider resource described by host, if any.
+// The returned Match's string fields alias host.
+func Classify(host string) (model.Match, bool) {
+	host = trimTrailingDots(host)
+	if host == "" || len(host) > 253 {
+		return model.Match{}, false
+	}
+	// Wildcard certificates never describe a concrete resource.
+	if host[0] == '*' {
+		return model.Match{}, false
+	}
+	if hasUpper(host) {
+		host = strings.ToLower(host)
+	}
+	if !validHost(host) {
+		return model.Match{}, false
+	}
+
+	s := newScanner(host)
+	if s.n < 2 {
+		return model.Match{}, false
+	}
+
+	switch {
+	case strings.HasSuffix(host, ".amazonaws.com"):
+		return classifyAWS(&s)
+	case strings.HasSuffix(host, ".core.windows.net"):
+		return classifyAzureCore(&s)
+	case strings.HasSuffix(host, ".database.windows.net"):
+		return classifyAzureDatabase(&s)
+	case strings.HasSuffix(host, ".database.azure.com"):
+		return classifyAzureFlexibleServer(&s)
+	case strings.HasSuffix(host, ".documents.azure.com"):
+		return classifyAzureDocuments(&s)
+	case strings.HasSuffix(host, ".cosmos.azure.com"):
+		return classifyAzureCosmos(&s)
+	case strings.HasSuffix(host, ".cache.windows.net"):
+		return classifyAzureRedis(&s)
+	case strings.HasSuffix(host, ".redis.azure.net"):
+		return classifyAzureManagedRedis(&s)
+	case strings.HasSuffix(host, ".search.windows.net"):
+		return classifyAzureSearch(&s)
+	case strings.HasSuffix(host, ".storage.googleapis.com"):
+		return classifyGCS(&s)
+	case strings.HasSuffix(host, ".firebaseio.com"):
+		return classifyFirebaseIO(&s)
+	case strings.HasSuffix(host, ".firebasedatabase.app"):
+		return classifyFirebaseDB(&s)
+	case strings.HasSuffix(host, ".appspot.com"):
+		return classifyAppspot(&s)
+	case strings.HasSuffix(host, ".r2.cloudflarestorage.com"):
+		return classifyR2(&s)
+	case strings.HasSuffix(host, ".r2.dev"):
+		return classifyR2Dev(&s)
+	case strings.HasSuffix(host, ".digitaloceanspaces.com"):
+		return classifyDigitalOceanSpaces(&s)
+	case strings.HasSuffix(host, ".ondigitalocean.com"):
+		return classifyDigitalOceanDB(&s)
+	case strings.HasSuffix(host, ".supabase.co"):
+		return classifySupabase(&s)
+	case strings.HasSuffix(host, ".supabase.com"):
+		return classifySupabasePooler(&s)
+	case strings.HasSuffix(host, ".customer-oci.com"):
+		return classifyOCIResource(&s)
+	case strings.HasSuffix(host, ".oraclecloud.com"):
+		return classifyOCIGeneric(&s)
+	}
+	return model.Match{}, false
+}
+
+// ShouldEnrich reports whether a non-matching hostname is worth a DNS/CNAME
+// lookup. It rejects wildcards, the certspotter unparsable placeholder, single
+// labels and obvious provider service apexes.
+func ShouldEnrich(host string) bool {
+	host = trimTrailingDots(host)
+	if len(host) < 3 || host[0] == '*' {
+		return false
+	}
+	if host[0] == '<' { // certspotter's "<unparsable>" placeholder
+		return false
+	}
+	if !strings.Contains(host, ".") {
+		return false
+	}
+	if hasUpper(host) {
+		host = strings.ToLower(host)
+	}
+	// Provider service apexes never yield a customer resource via CNAME.
+	switch {
+	case host == "amazonaws.com", strings.HasSuffix(host, ".amazonaws.com"):
+		return false
+	case host == "core.windows.net", strings.HasSuffix(host, ".windows.net"):
+		return false
+	case host == "azure.com", strings.HasSuffix(host, ".azure.com"):
+		return false
+	case host == "googleapis.com", strings.HasSuffix(host, ".googleapis.com"):
+		return false
+	case host == "cloudflarestorage.com", host == "r2.dev":
+		return false
+	case strings.HasSuffix(host, ".digitaloceanspaces.com"):
+		return false
+	case strings.HasSuffix(host, ".supabase.co"), strings.HasSuffix(host, ".supabase.com"):
+		return false
+	case strings.HasSuffix(host, ".oraclecloud.com"), strings.HasSuffix(host, ".customer-oci.com"):
+		return false
+	}
+	return true
+}

internal/classify/classify_test.go

@@ -0,0 +1,243 @@
+package classify
+
+import (
+	"strings"
+	"testing"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+type want struct {
+	provider         model.Provider
+	service          model.Service
+	resource         string
+	region           string
+	resourceSpecific bool
+	semantic         model.PublicSemantic
+}
+
+func run(t *testing.T, host string, w *want) {
+	t.Helper()
+	m, ok := Classify(host)
+	if w == nil {
+		if ok {
+			t.Fatalf("Classify(%q) matched %+v, want no match", host, m)
+		}
+		return
+	}
+	if !ok {
+		t.Fatalf("Classify(%q) did not match, want %+v", host, *w)
+	}
+	if m.Provider != w.provider || m.Service != w.service {
+		t.Fatalf("Classify(%q) = %s/%s, want %s/%s", host, m.Provider, m.Service, w.provider, w.service)
+	}
+	if w.resource != "" && m.Resource != w.resource {
+		t.Fatalf("Classify(%q) resource = %q, want %q", host, m.Resource, w.resource)
+	}
+	if w.region != "" && m.Region != w.region {
+		t.Fatalf("Classify(%q) region = %q, want %q", host, m.Region, w.region)
+	}
+	if m.ResourceSpecific != w.resourceSpecific {
+		t.Fatalf("Classify(%q) resourceSpecific = %v, want %v", host, m.ResourceSpecific, w.resourceSpecific)
+	}
+	if w.semantic != 0 && m.PublicSemantic != w.semantic {
+		t.Fatalf("Classify(%q) semantic = %s, want %s", host, m.PublicSemantic, w.semantic)
+	}
+}
+
+func TestClassifyMatches(t *testing.T) {
+	cases := []struct {
+		host string
+		w    want
+	}{
+		// AWS S3
+		{"bucket.s3.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+		{"bucket.s3.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3.dualstack.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3-website.eu-west-2.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-2", true, 0}},
+		{"bucket.s3-website-us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+		{"ap-123456.s3-accesspoint.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "ap-123456", "us-east-1", true, 0}},
+		{"x.s3-accesspoint.dualstack.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
+		{"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
+		{"x.mrap.accesspoint.s3-global.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "", true, 0}},
+		{"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}},
+		// AWS databases
+		{"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.proxy-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.cluster-abc.us-east-1.docdb.amazonaws.com", want{model.ProviderAWS, model.ServiceDocumentDB, "mydb", "us-east-1", true, 0}},
+		{"cluster.id.eu-central-1.redshift.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
+		{"cluster.id.eu-central-1.redshift-serverless.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
+		{"search-example-abc.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "search-example-abc", "eu-west-1", true, model.PublicSemanticPublicEndpoint}},
+		{"vpc-example.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "vpc-example", "eu-west-1", true, model.PublicSemanticVPCEndpoint}},
+		{"mydb.cluster-abc.us-east-1.neptune.amazonaws.com", want{model.ProviderAWS, model.ServiceNeptune, "mydb", "us-east-1", true, 0}},
+		{"clustercfg.my.xxxx.memorydb.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceMemoryDB, "clustercfg", "us-east-1", true, 0}},
+		{"mycluster.abc.0001.use1.cache.amazonaws.com", want{model.ProviderAWS, model.ServiceElastiCache, "mycluster", "use1", true, 0}},
+		// Azure
+		{"myacct.blob.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
+		{"myacct.dfs.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
+		{"myacct.file.core.windows.net", want{model.ProviderAzure, model.ServiceFileStorage, "myacct", "", true, 0}},
+		{"myacct.queue.core.windows.net", want{model.ProviderAzure, model.ServiceQueue, "myacct", "", true, 0}},
+		{"myacct.table.core.windows.net", want{model.ProviderAzure, model.ServiceTableStorage, "myacct", "", true, 0}},
+		{"myacct.web.core.windows.net", want{model.ProviderAzure, model.ServiceWebApp, "myacct", "", true, 0}},
+		{"myserver.database.windows.net", want{model.ProviderAzure, model.ServiceDatabase, "myserver", "", true, 0}},
+		{"myserver.postgres.database.azure.com", want{model.ProviderAzure, model.ServicePostgres, "myserver", "", true, 0}},
+		{"myserver.mysql.database.azure.com", want{model.ProviderAzure, model.ServiceMySQL, "myserver", "", true, 0}},
+		{"mycosmos.documents.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
+		{"mycosmos.mongo.cosmos.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
+		{"myredis.redis.cache.windows.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
+		{"myredis.redis.azure.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
+		{"mysearch.search.windows.net", want{model.ProviderAzure, model.ServiceSearch, "mysearch", "", true, 0}},
+		// GCP
+		{"bucket.storage.googleapis.com", want{model.ProviderGCP, model.ServiceObjectStorage, "bucket", "", true, 0}},
+		{"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}},
+		{"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}},
+		{"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}},
+		// Cloudflare
+		{"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}},
+		{"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}},
+		// DigitalOcean
+		{"bucket.fra1.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
+		{"bucket.fra1.cdn.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
+		{"mydb.db.ondigitalocean.com", want{model.ProviderDigitalOcean, model.ServiceDatabase, "mydb", "", true, 0}},
+		// Supabase
+		{"abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServiceDatabase, "abcdefgh", "", true, 0}},
+		{"db.abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServicePostgres, "abcdefgh", "", true, 0}},
+		{"aws-0-us-east-1.pooler.supabase.com", want{model.ProviderSupabase, model.ServicePostgres, "aws-0-us-east-1", "", true, 0}},
+		// OCI
+		{"myns.objectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
+		{"myns.swiftobjectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
+		{"objectstorage.us-ashburn-1.oraclecloud.com", want{model.ProviderOracle, model.ServiceObjectStorage, "", "us-ashburn-1", false, 0}},
+		// normalization
+		{"BUCKET.S3.EU-CENTRAL-1.AMAZONAWS.COM", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3.amazonaws.com.", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+	}
+	for _, c := range cases {
+		w := c.w
+		run(t, c.host, &w)
+	}
+}
+
+func TestClassifyRejects(t *testing.T) {
+	rejects := []string{
+		// AWS generic endpoints / non-resources
+		"amazonaws.com",
+		"s3.amazonaws.com",
+		"s3.eu-central-1.amazonaws.com",
+		"s3.dualstack.eu-central-1.amazonaws.com",
+		"s3-accesspoint.us-east-1.amazonaws.com",
+		"dynamodb.eu-central-1.amazonaws.com",
+		"redshift.eu-central-1.amazonaws.com",
+		"rds.us-east-1.amazonaws.com",
+		"docdb.us-east-1.amazonaws.com",
+		"neptune.us-east-1.amazonaws.com",
+		"es.us-east-1.amazonaws.com",
+		"memorydb.us-east-1.amazonaws.com",
+		"cache.amazonaws.com",
+		"use1.cache.amazonaws.com",
+		"s3-global.amazonaws.com",
+		// Azure bare service endpoints
+		"blob.core.windows.net",
+		"database.windows.net",
+		"postgres.database.azure.com",
+		"documents.azure.com",
+		"mongo.cosmos.azure.com",
+		"redis.cache.windows.net",
+		"redis.azure.net",
+		"search.windows.net",
+		// GCP bare service endpoints
+		"storage.googleapis.com",
+		"firebaseio.com",
+		"firebasedatabase.app",
+		"appspot.com",
+		// Cloudflare
+		"r2.cloudflarestorage.com",
+		"r2.dev",
+		// DigitalOcean
+		"fra1.digitaloceanspaces.com",
+		"db.ondigitalocean.com",
+		// Supabase
+		"supabase.co",
+		"pooler.supabase.com",
+		// OCI
+		"oraclecloud.com",
+		"customer-oci.com",
+		// wildcards
+		"*.example.com",
+		"*.blob.core.windows.net",
+		"*.s3.amazonaws.com",
+		// normal hostnames
+		"www.example.com",
+		"foo.bar.example.org",
+		"localhost",
+		"",
+		"-",
+		// invalid / too long
+		"foo bar.com",
+		strings.Repeat("a.", 130) + "com",
+	}
+	for _, host := range rejects {
+		run(t, host, nil)
+	}
+}
+
+func TestShouldEnrich(t *testing.T) {
+	yes := []string{"files.example.com", "db.example.com", "a.b.c.example.org"}
+	no := []string{"*.example.com", "s3.amazonaws.com", "foo.s3.amazonaws.com", "storage.googleapis.com", "<unparsable>", ""}
+	for _, h := range yes {
+		if !ShouldEnrich(h) {
+			t.Errorf("ShouldEnrich(%q) = false, want true", h)
+		}
+	}
+	for _, h := range no {
+		if ShouldEnrich(h) {
+			t.Errorf("ShouldEnrich(%q) = true, want false", h)
+		}
+	}
+}
+
+var benchInputs = []string{
+	"www.example.com",
+	"foo.bar.example.org",
+	"bucket.s3.eu-central-1.amazonaws.com",
+	"foo.blob.core.windows.net",
+	"db.example.supabase.co",
+	"project.firebaseio.com",
+	"search-example-abc.eu-west-1.es.amazonaws.com",
+}
+
+func BenchmarkClassifyAWS(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("bucket.s3.eu-central-1.amazonaws.com")
+	}
+}
+
+func BenchmarkClassifyAzure(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("foo.blob.core.windows.net")
+	}
+}
+
+func BenchmarkRejectNormalDomain(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("www.example.com")
+	}
+}
+
+func BenchmarkClassifyMixed(b *testing.B) {
+	allocs := testing.AllocsPerRun(1000, func() {
+		for _, host := range benchInputs {
+			Classify(host)
+		}
+	})
+	if allocs != 0 {
+		b.Fatalf("Classify allocates %.2f allocs/op, want 0", allocs)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, host := range benchInputs {
+			Classify(host)
+		}
+	}
+}

internal/classify/cloudflare.go

@@ -0,0 +1,34 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyR2 handles <account-id>.r2.cloudflarestorage.com.
+func classifyR2(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "cloudflarestorage" || s.label(n-3) != "r2" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderCloudflare,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}
+
+// classifyR2Dev handles <account>.r2.dev.  The hostname form is explicitly a
+// public development/public URL, which is why it carries PublicSemanticPublicURL
+// while still not asserting that data is readable.
+func classifyR2Dev(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "dev" || s.label(n-2) != "r2" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderCloudflare,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		PublicSemantic:   model.PublicSemanticPublicURL,
+		Resource:         s.label(0),
+	}, true
+}

internal/classify/digitalocean.go

@@ -0,0 +1,39 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyDigitalOceanSpaces handles
+// <bucket>.<region>.digitaloceanspaces.com and the cdn variant.
+func classifyDigitalOceanSpaces(s *scanner) (model.Match, bool) {
+	n := s.n
+	if s.label(n-1) != "com" || s.label(n-2) != "digitaloceanspaces" {
+		return model.Match{}, false
+	}
+	// <bucket>.<region>.digitaloceanspaces.com (n==4) or the cdn variant (n==5)
+	if n != 4 && !(n == 5 && s.label(2) == "cdn") {
+		return model.Match{}, false
+	}
+	bucket := s.label(0)
+	region := s.label(1)
+	return model.Match{
+		Provider:         model.ProviderDigitalOcean,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         bucket,
+	}, true
+}
+
+// classifyDigitalOceanDB handles <name>.db.ondigitalocean.com.
+func classifyDigitalOceanDB(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "ondigitalocean" || s.label(n-3) != "db" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderDigitalOcean,
+		Service:          model.ServiceDatabase,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}

internal/classify/gcp.go

@@ -0,0 +1,49 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+func gcp(service model.Service, resource, region string) model.Match {
+	return model.Match{
+		Provider:         model.ProviderGCP,
+		Service:          service,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+// classifyGCS handles <bucket>.storage.googleapis.com.
+func classifyGCS(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "googleapis" || s.label(n-3) != "storage" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceObjectStorage, s.label(0), ""), true
+}
+
+// classifyFirebaseIO handles <database>.firebaseio.com.
+func classifyFirebaseIO(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "firebaseio" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceFirebaseDB, s.label(0), ""), true
+}
+
+// classifyFirebaseDB handles <database>.<region>.firebasedatabase.app.
+func classifyFirebaseDB(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "app" || s.label(n-2) != "firebasedatabase" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceFirebaseDB, s.label(0), s.label(1)), true
+}
+
+// classifyAppspot handles <project>.appspot.com.
+func classifyAppspot(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "appspot" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceWebApp, s.label(0), ""), true
+}

internal/classify/oracle.go

@@ -0,0 +1,44 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyOCIResource handles
+// <namespace>.objectstorage.<region>.oci.customer-oci.com and the swift variant.
+// The namespace is in the hostname, so it is resource specific.
+func classifyOCIResource(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 6 || s.label(n-1) != "com" || s.label(n-2) != "customer-oci" || s.label(n-3) != "oci" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 5)
+	if svc != "objectstorage" && svc != "swiftobjectstorage" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderOracle,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Region:           s.label(n - 4),
+		Resource:         s.label(0),
+	}, true
+}
+
+// classifyOCIGeneric handles objectstorage.<region>.oraclecloud.com. The bucket
+// name lives in the URL path rather than the hostname, so resource_specific is
+// false and these normally do not become findings.
+func classifyOCIGeneric(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "oraclecloud" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 4)
+	if svc != "objectstorage" && svc != "swiftobjectstorage" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderOracle,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: false,
+		Region:           s.label(n - 3),
+	}, true
+}

internal/classify/supabase.go

@@ -0,0 +1,45 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifySupabase handles <project-ref>.supabase.co and
+// db.<project-ref>.supabase.co (the latter classified as PostgreSQL).
+func classifySupabase(s *scanner) (model.Match, bool) {
+	n := s.n
+	if s.label(n-1) != "co" || s.label(n-2) != "supabase" {
+		return model.Match{}, false
+	}
+	switch n {
+	case 3:
+		return model.Match{
+			Provider:         model.ProviderSupabase,
+			Service:          model.ServiceDatabase,
+			ResourceSpecific: true,
+			Resource:         s.label(0),
+		}, true
+	case 4:
+		if s.label(0) == "db" {
+			return model.Match{
+				Provider:         model.ProviderSupabase,
+				Service:          model.ServicePostgres,
+				ResourceSpecific: true,
+				Resource:         s.label(1),
+			}, true
+		}
+	}
+	return model.Match{}, false
+}
+
+// classifySupabasePooler handles aws-<index>-<region>.pooler.supabase.com.
+func classifySupabasePooler(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "supabase" || s.label(n-3) != "pooler" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderSupabase,
+		Service:          model.ServicePostgres,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}