ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
This commit is contained in:
StasonJatham committed
1 parent
9e78a114ac
commit
e8f7d6bfdf
118 files changed
+16473
No files matched your search
@@ -0,0 +1,243 @@
|
||||
package classify
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
type want struct {
|
||||
provider model.Provider
|
||||
service model.Service
|
||||
resource string
|
||||
region string
|
||||
resourceSpecific bool
|
||||
semantic model.PublicSemantic
|
||||
}
|
||||
|
||||
func run(t *testing.T, host string, w *want) {
|
||||
t.Helper()
|
||||
m, ok := Classify(host)
|
||||
if w == nil {
|
||||
if ok {
|
||||
t.Fatalf("Classify(%q) matched %+v, want no match", host, m)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
t.Fatalf("Classify(%q) did not match, want %+v", host, *w)
|
||||
}
|
||||
if m.Provider != w.provider || m.Service != w.service {
|
||||
t.Fatalf("Classify(%q) = %s/%s, want %s/%s", host, m.Provider, m.Service, w.provider, w.service)
|
||||
}
|
||||
if w.resource != "" && m.Resource != w.resource {
|
||||
t.Fatalf("Classify(%q) resource = %q, want %q", host, m.Resource, w.resource)
|
||||
}
|
||||
if w.region != "" && m.Region != w.region {
|
||||
t.Fatalf("Classify(%q) region = %q, want %q", host, m.Region, w.region)
|
||||
}
|
||||
if m.ResourceSpecific != w.resourceSpecific {
|
||||
t.Fatalf("Classify(%q) resourceSpecific = %v, want %v", host, m.ResourceSpecific, w.resourceSpecific)
|
||||
}
|
||||
if w.semantic != 0 && m.PublicSemantic != w.semantic {
|
||||
t.Fatalf("Classify(%q) semantic = %s, want %s", host, m.PublicSemantic, w.semantic)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyMatches(t *testing.T) {
|
||||
cases := []struct {
|
||||
host string
|
||||
w want
|
||||
}{
|
||||
// AWS S3
|
||||
{"bucket.s3.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
|
||||
{"bucket.s3.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
|
||||
{"bucket.s3.dualstack.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
|
||||
{"bucket.s3-website.eu-west-2.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-2", true, 0}},
|
||||
{"bucket.s3-website-us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
|
||||
{"ap-123456.s3-accesspoint.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "ap-123456", "us-east-1", true, 0}},
|
||||
{"x.s3-accesspoint.dualstack.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
|
||||
{"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
|
||||
{"x.mrap.accesspoint.s3-global.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "", true, 0}},
|
||||
{"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}},
|
||||
// AWS databases
|
||||
{"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
|
||||
{"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
|
||||
{"mydb.proxy-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
|
||||
{"mydb.cluster-abc.us-east-1.docdb.amazonaws.com", want{model.ProviderAWS, model.ServiceDocumentDB, "mydb", "us-east-1", true, 0}},
|
||||
{"cluster.id.eu-central-1.redshift.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
|
||||
{"cluster.id.eu-central-1.redshift-serverless.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
|
||||
{"search-example-abc.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "search-example-abc", "eu-west-1", true, model.PublicSemanticPublicEndpoint}},
|
||||
{"vpc-example.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "vpc-example", "eu-west-1", true, model.PublicSemanticVPCEndpoint}},
|
||||
{"mydb.cluster-abc.us-east-1.neptune.amazonaws.com", want{model.ProviderAWS, model.ServiceNeptune, "mydb", "us-east-1", true, 0}},
|
||||
{"clustercfg.my.xxxx.memorydb.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceMemoryDB, "clustercfg", "us-east-1", true, 0}},
|
||||
{"mycluster.abc.0001.use1.cache.amazonaws.com", want{model.ProviderAWS, model.ServiceElastiCache, "mycluster", "use1", true, 0}},
|
||||
// Azure
|
||||
{"myacct.blob.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
|
||||
{"myacct.dfs.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
|
||||
{"myacct.file.core.windows.net", want{model.ProviderAzure, model.ServiceFileStorage, "myacct", "", true, 0}},
|
||||
{"myacct.queue.core.windows.net", want{model.ProviderAzure, model.ServiceQueue, "myacct", "", true, 0}},
|
||||
{"myacct.table.core.windows.net", want{model.ProviderAzure, model.ServiceTableStorage, "myacct", "", true, 0}},
|
||||
{"myacct.web.core.windows.net", want{model.ProviderAzure, model.ServiceWebApp, "myacct", "", true, 0}},
|
||||
{"myserver.database.windows.net", want{model.ProviderAzure, model.ServiceDatabase, "myserver", "", true, 0}},
|
||||
{"myserver.postgres.database.azure.com", want{model.ProviderAzure, model.ServicePostgres, "myserver", "", true, 0}},
|
||||
{"myserver.mysql.database.azure.com", want{model.ProviderAzure, model.ServiceMySQL, "myserver", "", true, 0}},
|
||||
{"mycosmos.documents.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
|
||||
{"mycosmos.mongo.cosmos.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
|
||||
{"myredis.redis.cache.windows.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
|
||||
{"myredis.redis.azure.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
|
||||
{"mysearch.search.windows.net", want{model.ProviderAzure, model.ServiceSearch, "mysearch", "", true, 0}},
|
||||
// GCP
|
||||
{"bucket.storage.googleapis.com", want{model.ProviderGCP, model.ServiceObjectStorage, "bucket", "", true, 0}},
|
||||
{"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}},
|
||||
{"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}},
|
||||
{"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}},
|
||||
// Cloudflare
|
||||
{"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}},
|
||||
{"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}},
|
||||
// DigitalOcean
|
||||
{"bucket.fra1.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
|
||||
{"bucket.fra1.cdn.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
|
||||
{"mydb.db.ondigitalocean.com", want{model.ProviderDigitalOcean, model.ServiceDatabase, "mydb", "", true, 0}},
|
||||
// Supabase
|
||||
{"abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServiceDatabase, "abcdefgh", "", true, 0}},
|
||||
{"db.abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServicePostgres, "abcdefgh", "", true, 0}},
|
||||
{"aws-0-us-east-1.pooler.supabase.com", want{model.ProviderSupabase, model.ServicePostgres, "aws-0-us-east-1", "", true, 0}},
|
||||
// OCI
|
||||
{"myns.objectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
|
||||
{"myns.swiftobjectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
|
||||
{"objectstorage.us-ashburn-1.oraclecloud.com", want{model.ProviderOracle, model.ServiceObjectStorage, "", "us-ashburn-1", false, 0}},
|
||||
// normalization
|
||||
{"BUCKET.S3.EU-CENTRAL-1.AMAZONAWS.COM", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
|
||||
{"bucket.s3.amazonaws.com.", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
w := c.w
|
||||
run(t, c.host, &w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyRejects(t *testing.T) {
|
||||
rejects := []string{
|
||||
// AWS generic endpoints / non-resources
|
||||
"amazonaws.com",
|
||||
"s3.amazonaws.com",
|
||||
"s3.eu-central-1.amazonaws.com",
|
||||
"s3.dualstack.eu-central-1.amazonaws.com",
|
||||
"s3-accesspoint.us-east-1.amazonaws.com",
|
||||
"dynamodb.eu-central-1.amazonaws.com",
|
||||
"redshift.eu-central-1.amazonaws.com",
|
||||
"rds.us-east-1.amazonaws.com",
|
||||
"docdb.us-east-1.amazonaws.com",
|
||||
"neptune.us-east-1.amazonaws.com",
|
||||
"es.us-east-1.amazonaws.com",
|
||||
"memorydb.us-east-1.amazonaws.com",
|
||||
"cache.amazonaws.com",
|
||||
"use1.cache.amazonaws.com",
|
||||
"s3-global.amazonaws.com",
|
||||
// Azure bare service endpoints
|
||||
"blob.core.windows.net",
|
||||
"database.windows.net",
|
||||
"postgres.database.azure.com",
|
||||
"documents.azure.com",
|
||||
"mongo.cosmos.azure.com",
|
||||
"redis.cache.windows.net",
|
||||
"redis.azure.net",
|
||||
"search.windows.net",
|
||||
// GCP bare service endpoints
|
||||
"storage.googleapis.com",
|
||||
"firebaseio.com",
|
||||
"firebasedatabase.app",
|
||||
"appspot.com",
|
||||
// Cloudflare
|
||||
"r2.cloudflarestorage.com",
|
||||
"r2.dev",
|
||||
// DigitalOcean
|
||||
"fra1.digitaloceanspaces.com",
|
||||
"db.ondigitalocean.com",
|
||||
// Supabase
|
||||
"supabase.co",
|
||||
"pooler.supabase.com",
|
||||
// OCI
|
||||
"oraclecloud.com",
|
||||
"customer-oci.com",
|
||||
// wildcards
|
||||
"*.example.com",
|
||||
"*.blob.core.windows.net",
|
||||
"*.s3.amazonaws.com",
|
||||
// normal hostnames
|
||||
"www.example.com",
|
||||
"foo.bar.example.org",
|
||||
"localhost",
|
||||
"",
|
||||
"-",
|
||||
// invalid / too long
|
||||
"foo bar.com",
|
||||
strings.Repeat("a.", 130) + "com",
|
||||
}
|
||||
for _, host := range rejects {
|
||||
run(t, host, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldEnrich(t *testing.T) {
|
||||
yes := []string{"files.example.com", "db.example.com", "a.b.c.example.org"}
|
||||
no := []string{"*.example.com", "s3.amazonaws.com", "foo.s3.amazonaws.com", "storage.googleapis.com", "<unparsable>", ""}
|
||||
for _, h := range yes {
|
||||
if !ShouldEnrich(h) {
|
||||
t.Errorf("ShouldEnrich(%q) = false, want true", h)
|
||||
}
|
||||
}
|
||||
for _, h := range no {
|
||||
if ShouldEnrich(h) {
|
||||
t.Errorf("ShouldEnrich(%q) = true, want false", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var benchInputs = []string{
|
||||
"www.example.com",
|
||||
"foo.bar.example.org",
|
||||
"bucket.s3.eu-central-1.amazonaws.com",
|
||||
"foo.blob.core.windows.net",
|
||||
"db.example.supabase.co",
|
||||
"project.firebaseio.com",
|
||||
"search-example-abc.eu-west-1.es.amazonaws.com",
|
||||
}
|
||||
|
||||
func BenchmarkClassifyAWS(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
Classify("bucket.s3.eu-central-1.amazonaws.com")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkClassifyAzure(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
Classify("foo.blob.core.windows.net")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRejectNormalDomain(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
Classify("www.example.com")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkClassifyMixed(b *testing.B) {
|
||||
allocs := testing.AllocsPerRun(1000, func() {
|
||||
for _, host := range benchInputs {
|
||||
Classify(host)
|
||||
}
|
||||
})
|
||||
if allocs != 0 {
|
||||
b.Fatalf("Classify allocates %.2f allocs/op, want 0", allocs)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
for _, host := range benchInputs {
|
||||
Classify(host)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user