Files
bucketeer/internal/classify/aws.go
T
StasonJatham e8f7d6bfdf ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path
2026-09-22 13:13:10 +02:00

149 lines
4.7 KiB
Go

package classify
import (
"strings"
"tea.karlcom.de/karl/bucketeer/internal/model"
)
func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match {
return model.Match{
Provider: p,
Service: s,
PublicSemantic: sem,
ResourceSpecific: true,
Region: region,
Resource: resource,
}
}
func classifyAWS(s *scanner) (model.Match, bool) {
n := s.n
if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {
return model.Match{}, false
}
if m, ok := classifyAWSS3(s); ok {
return m, true
}
// Service-after-region forms: <resource>.<region>.<service>.amazonaws.com
if n >= 5 {
switch s.label(n - 3) {
case "rds":
return awsResourceAfterRegion(s, model.ServiceRDS)
case "docdb":
return awsResourceAfterRegion(s, model.ServiceDocumentDB)
case "neptune":
return awsResourceAfterRegion(s, model.ServiceNeptune)
case "redshift", "redshift-serverless":
return awsResourceAfterRegion(s, model.ServiceRedshift)
case "es":
first := s.label(0)
switch {
case strings.HasPrefix(first, "search-"):
return aws(model.ProviderAWS, model.ServiceOpenSearch,
model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true
case strings.HasPrefix(first, "vpc-"):
return aws(model.ProviderAWS, model.ServiceOpenSearch,
model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true
}
case "cache":
return aws(model.ProviderAWS, model.ServiceElastiCache,
model.PublicSemanticNone, s.label(0), s.label(n-4)), true
}
}
// Service-before-region form: <resource>.memorydb.<region>.amazonaws.com
if n >= 5 && s.label(n-4) == "memorydb" {
return aws(model.ProviderAWS, model.ServiceMemoryDB,
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
}
return model.Match{}, false
}
// awsResourceAfterRegion matches <resource>.<region>.<service>.amazonaws.com
// and rejects the bare <service>.<region>.amazonaws.com service endpoint.
func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) {
n := s.n
if n < 5 {
return model.Match{}, false
}
resource := s.label(0)
if resource == s.label(n-3) {
return model.Match{}, false
}
return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true
}
func classifyAWSS3(s *scanner) (model.Match, bool) {
n := s.n
// Access points: <ap>.s3-accesspoint[-fips][.dualstack].<region>.amazonaws.com
if n >= 5 {
svcIdx := -1
switch {
case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips":
svcIdx = n - 4
case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint":
svcIdx = n - 5
}
if svcIdx > 0 {
res := s.label(0)
if res != "s3-accesspoint" && res != "s3-accesspoint-fips" {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, res, s.label(n-3)), true
}
}
}
// Multi-region access point: <res>.mrap.accesspoint.s3-global.amazonaws.com
if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" &&
s.label(n-3) == "s3-global" {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), ""), true
}
// S3 Express: <bucket>.s3express-<zone>.<region>.amazonaws.com
if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
}
// S3 website: <bucket>.s3-website.<region>.amazonaws.com
if n >= 5 && s.label(n-4) == "s3-website" {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
}
// S3 website (region in the service label): <bucket>.s3-website-<region>.amazonaws.com
if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), ""), true
}
// Plain S3 without region: <bucket>.s3.amazonaws.com
if n == 4 && s.label(n-3) == "s3" {
res := s.label(0)
if res == "s3" {
return model.Match{}, false
}
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true
}
// Plain S3 with region: <bucket>.s3.<region>.amazonaws.com
if n >= 5 && s.label(n-4) == "s3" {
res := s.label(0)
if res == "s3" {
return model.Match{}, false
}
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
}
// Dualstack S3: <bucket>.s3.dualstack.<region>.amazonaws.com
if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" {
res := s.label(0)
if res == "s3" {
return model.Match{}, false
}
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
}
return model.Match{}, false
}