StasonJatham
816fa572df
MRAP aliases (<alias>.mrap.accesspoint.s3-global.amazonaws.com) were 91,560 of 91,561 S3 findings: AWS-generated, not customer buckets, never publicly resolvable, and they monopolised pipeline capacity. CloudFront distributions are CDN endpoints, not buckets or databases. Both are now excluded so real buckets and databases get the capacity.
246 lines
11 KiB
Go
246 lines
11 KiB
Go
package classify
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
)
|
|
|
|
type want struct {
|
|
provider model.Provider
|
|
service model.Service
|
|
resource string
|
|
region string
|
|
resourceSpecific bool
|
|
semantic model.PublicSemantic
|
|
}
|
|
|
|
func run(t *testing.T, host string, w *want) {
|
|
t.Helper()
|
|
m, ok := Classify(host)
|
|
if w == nil {
|
|
if ok {
|
|
t.Fatalf("Classify(%q) matched %+v, want no match", host, m)
|
|
}
|
|
return
|
|
}
|
|
if !ok {
|
|
t.Fatalf("Classify(%q) did not match, want %+v", host, *w)
|
|
}
|
|
if m.Provider != w.provider || m.Service != w.service {
|
|
t.Fatalf("Classify(%q) = %s/%s, want %s/%s", host, m.Provider, m.Service, w.provider, w.service)
|
|
}
|
|
if w.resource != "" && m.Resource != w.resource {
|
|
t.Fatalf("Classify(%q) resource = %q, want %q", host, m.Resource, w.resource)
|
|
}
|
|
if w.region != "" && m.Region != w.region {
|
|
t.Fatalf("Classify(%q) region = %q, want %q", host, m.Region, w.region)
|
|
}
|
|
if m.ResourceSpecific != w.resourceSpecific {
|
|
t.Fatalf("Classify(%q) resourceSpecific = %v, want %v", host, m.ResourceSpecific, w.resourceSpecific)
|
|
}
|
|
if w.semantic != 0 && m.PublicSemantic != w.semantic {
|
|
t.Fatalf("Classify(%q) semantic = %s, want %s", host, m.PublicSemantic, w.semantic)
|
|
}
|
|
}
|
|
|
|
func TestClassifyMatches(t *testing.T) {
|
|
cases := []struct {
|
|
host string
|
|
w want
|
|
}{
|
|
// AWS S3
|
|
{"bucket.s3.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
|
|
{"bucket.s3.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
|
|
{"bucket.s3.dualstack.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
|
|
{"bucket.s3-website.eu-west-2.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-2", true, 0}},
|
|
{"bucket.s3-website-us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
|
|
{"ap-123456.s3-accesspoint.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "ap-123456", "us-east-1", true, 0}},
|
|
{"x.s3-accesspoint.dualstack.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
|
|
{"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
|
|
{"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}},
|
|
{"bucket-01.s3-eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket-01", "eu-west-1", true, 0}},
|
|
// AWS databases
|
|
{"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
|
|
{"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
|
|
{"mydb.proxy-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
|
|
{"mydb.cluster-abc.us-east-1.docdb.amazonaws.com", want{model.ProviderAWS, model.ServiceDocumentDB, "mydb", "us-east-1", true, 0}},
|
|
{"cluster.id.eu-central-1.redshift.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
|
|
{"cluster.id.eu-central-1.redshift-serverless.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
|
|
{"search-example-abc.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "search-example-abc", "eu-west-1", true, model.PublicSemanticPublicEndpoint}},
|
|
{"vpc-example.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "vpc-example", "eu-west-1", true, model.PublicSemanticVPCEndpoint}},
|
|
{"mydb.cluster-abc.us-east-1.neptune.amazonaws.com", want{model.ProviderAWS, model.ServiceNeptune, "mydb", "us-east-1", true, 0}},
|
|
{"clustercfg.my.xxxx.memorydb.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceMemoryDB, "clustercfg", "us-east-1", true, 0}},
|
|
{"mycluster.abc.0001.use1.cache.amazonaws.com", want{model.ProviderAWS, model.ServiceElastiCache, "mycluster", "use1", true, 0}},
|
|
// Azure
|
|
{"myacct.blob.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
|
|
{"myacct.dfs.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
|
|
{"myacct.file.core.windows.net", want{model.ProviderAzure, model.ServiceFileStorage, "myacct", "", true, 0}},
|
|
{"myacct.queue.core.windows.net", want{model.ProviderAzure, model.ServiceQueue, "myacct", "", true, 0}},
|
|
{"myacct.table.core.windows.net", want{model.ProviderAzure, model.ServiceTableStorage, "myacct", "", true, 0}},
|
|
{"myacct.web.core.windows.net", want{model.ProviderAzure, model.ServiceWebApp, "myacct", "", true, 0}},
|
|
{"myserver.database.windows.net", want{model.ProviderAzure, model.ServiceDatabase, "myserver", "", true, 0}},
|
|
{"myserver.postgres.database.azure.com", want{model.ProviderAzure, model.ServicePostgres, "myserver", "", true, 0}},
|
|
{"myserver.mysql.database.azure.com", want{model.ProviderAzure, model.ServiceMySQL, "myserver", "", true, 0}},
|
|
{"mycosmos.documents.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
|
|
{"mycosmos.mongo.cosmos.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
|
|
{"myredis.redis.cache.windows.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
|
|
{"myredis.redis.azure.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
|
|
{"mysearch.search.windows.net", want{model.ProviderAzure, model.ServiceSearch, "mysearch", "", true, 0}},
|
|
// GCP
|
|
{"bucket.storage.googleapis.com", want{model.ProviderGCP, model.ServiceObjectStorage, "bucket", "", true, 0}},
|
|
{"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}},
|
|
{"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}},
|
|
{"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}},
|
|
{"d111111abcdef8.cloudfront.net", want{model.ProviderAWS, model.ServiceWebApp, "d111111abcdef8", "", false, model.PublicSemanticPublicEndpoint}},
|
|
// Cloudflare
|
|
{"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}},
|
|
{"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}},
|
|
// DigitalOcean
|
|
{"bucket.fra1.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
|
|
{"bucket.fra1.cdn.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
|
|
{"mydb.db.ondigitalocean.com", want{model.ProviderDigitalOcean, model.ServiceDatabase, "mydb", "", true, 0}},
|
|
// Supabase
|
|
{"abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServiceDatabase, "abcdefgh", "", true, 0}},
|
|
{"db.abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServicePostgres, "abcdefgh", "", true, 0}},
|
|
{"aws-0-us-east-1.pooler.supabase.com", want{model.ProviderSupabase, model.ServicePostgres, "aws-0-us-east-1", "", true, 0}},
|
|
// OCI
|
|
{"myns.objectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
|
|
{"myns.swiftobjectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
|
|
{"objectstorage.us-ashburn-1.oraclecloud.com", want{model.ProviderOracle, model.ServiceObjectStorage, "", "us-ashburn-1", false, 0}},
|
|
// normalization
|
|
{"BUCKET.S3.EU-CENTRAL-1.AMAZONAWS.COM", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
|
|
{"bucket.s3.amazonaws.com.", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
|
|
}
|
|
for _, c := range cases {
|
|
w := c.w
|
|
run(t, c.host, &w)
|
|
}
|
|
}
|
|
|
|
func TestClassifyRejects(t *testing.T) {
|
|
rejects := []string{
|
|
// AWS generic endpoints / non-resources
|
|
"amazonaws.com",
|
|
"s3.amazonaws.com",
|
|
"s3.eu-central-1.amazonaws.com",
|
|
"s3.dualstack.eu-central-1.amazonaws.com",
|
|
"s3-accesspoint.us-east-1.amazonaws.com",
|
|
"dynamodb.eu-central-1.amazonaws.com",
|
|
"redshift.eu-central-1.amazonaws.com",
|
|
"rds.us-east-1.amazonaws.com",
|
|
"docdb.us-east-1.amazonaws.com",
|
|
"neptune.us-east-1.amazonaws.com",
|
|
"es.us-east-1.amazonaws.com",
|
|
"memorydb.us-east-1.amazonaws.com",
|
|
"cache.amazonaws.com",
|
|
"use1.cache.amazonaws.com",
|
|
"s3-global.amazonaws.com",
|
|
"x.mrap.accesspoint.s3-global.amazonaws.com",
|
|
// Azure bare service endpoints
|
|
"blob.core.windows.net",
|
|
"database.windows.net",
|
|
"postgres.database.azure.com",
|
|
"documents.azure.com",
|
|
"mongo.cosmos.azure.com",
|
|
"redis.cache.windows.net",
|
|
"redis.azure.net",
|
|
"search.windows.net",
|
|
// GCP bare service endpoints
|
|
"storage.googleapis.com",
|
|
"firebaseio.com",
|
|
"firebasedatabase.app",
|
|
"appspot.com",
|
|
// Cloudflare
|
|
"r2.cloudflarestorage.com",
|
|
"r2.dev",
|
|
// DigitalOcean
|
|
"fra1.digitaloceanspaces.com",
|
|
"db.ondigitalocean.com",
|
|
// Supabase
|
|
"supabase.co",
|
|
"pooler.supabase.com",
|
|
// OCI
|
|
"oraclecloud.com",
|
|
"customer-oci.com",
|
|
// wildcards
|
|
"*.example.com",
|
|
"*.blob.core.windows.net",
|
|
"*.s3.amazonaws.com",
|
|
// normal hostnames
|
|
"www.example.com",
|
|
"foo.bar.example.org",
|
|
"localhost",
|
|
"",
|
|
"-",
|
|
// invalid / too long
|
|
"foo bar.com",
|
|
strings.Repeat("a.", 130) + "com",
|
|
}
|
|
for _, host := range rejects {
|
|
run(t, host, nil)
|
|
}
|
|
}
|
|
|
|
func TestShouldEnrich(t *testing.T) {
|
|
yes := []string{"files.example.com", "db.example.com", "a.b.c.example.org"}
|
|
no := []string{"*.example.com", "s3.amazonaws.com", "foo.s3.amazonaws.com", "storage.googleapis.com", "<unparsable>", ""}
|
|
for _, h := range yes {
|
|
if !ShouldEnrich(h) {
|
|
t.Errorf("ShouldEnrich(%q) = false, want true", h)
|
|
}
|
|
}
|
|
for _, h := range no {
|
|
if ShouldEnrich(h) {
|
|
t.Errorf("ShouldEnrich(%q) = true, want false", h)
|
|
}
|
|
}
|
|
}
|
|
|
|
var benchInputs = []string{
|
|
"www.example.com",
|
|
"foo.bar.example.org",
|
|
"bucket.s3.eu-central-1.amazonaws.com",
|
|
"foo.blob.core.windows.net",
|
|
"db.example.supabase.co",
|
|
"project.firebaseio.com",
|
|
"search-example-abc.eu-west-1.es.amazonaws.com",
|
|
}
|
|
|
|
func BenchmarkClassifyAWS(b *testing.B) {
|
|
for i := 0; i < b.N; i++ {
|
|
Classify("bucket.s3.eu-central-1.amazonaws.com")
|
|
}
|
|
}
|
|
|
|
func BenchmarkClassifyAzure(b *testing.B) {
|
|
for i := 0; i < b.N; i++ {
|
|
Classify("foo.blob.core.windows.net")
|
|
}
|
|
}
|
|
|
|
func BenchmarkRejectNormalDomain(b *testing.B) {
|
|
for i := 0; i < b.N; i++ {
|
|
Classify("www.example.com")
|
|
}
|
|
}
|
|
|
|
func BenchmarkClassifyMixed(b *testing.B) {
|
|
allocs := testing.AllocsPerRun(1000, func() {
|
|
for _, host := range benchInputs {
|
|
Classify(host)
|
|
}
|
|
})
|
|
if allocs != 0 {
|
|
b.Fatalf("Classify allocates %.2f allocs/op, want 0", allocs)
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
for _, host := range benchInputs {
|
|
Classify(host)
|
|
}
|
|
}
|
|
}
|