Files
bucketeer/internal/classify/aws.go
T
StasonJatham 62a7d2eb2e provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
  object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
  object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.

Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.

Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.

On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.

Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.

DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.

Verified live against real public resources:
  veteran1.nyc3.digitaloceanspaces.com            listable, entries
  bucket-01.s3-eu-west-1.amazonaws.com            listable, entries
  tartugeo-podcast.storage.googleapis.com         listable, entries
  eroticfeeldev.blob.core.windows.net/images      listable, entries
2026-09-22 14:47:49 +02:00

161 lines
5.2 KiB
Go

package classify
import (
"strings"
"tea.karlcom.de/karl/bucketeer/internal/model"
)
func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match {
return model.Match{
Provider: p,
Service: s,
PublicSemantic: sem,
ResourceSpecific: true,
Region: region,
Resource: resource,
}
}
func classifyAWS(s *scanner) (model.Match, bool) {
n := s.n
if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {
return model.Match{}, false
}
if m, ok := classifyAWSS3(s); ok {
return m, true
}
// Service-after-region forms: <resource>.<region>.<service>.amazonaws.com
if n >= 5 {
switch s.label(n - 3) {
case "rds":
return awsResourceAfterRegion(s, model.ServiceRDS)
case "docdb":
return awsResourceAfterRegion(s, model.ServiceDocumentDB)
case "neptune":
return awsResourceAfterRegion(s, model.ServiceNeptune)
case "redshift", "redshift-serverless":
return awsResourceAfterRegion(s, model.ServiceRedshift)
case "es":
first := s.label(0)
switch {
case strings.HasPrefix(first, "search-"):
return aws(model.ProviderAWS, model.ServiceOpenSearch,
model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true
case strings.HasPrefix(first, "vpc-"):
return aws(model.ProviderAWS, model.ServiceOpenSearch,
model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true
}
case "cache":
return aws(model.ProviderAWS, model.ServiceElastiCache,
model.PublicSemanticNone, s.label(0), s.label(n-4)), true
}
}
// Service-before-region form: <resource>.memorydb.<region>.amazonaws.com
if n >= 5 && s.label(n-4) == "memorydb" {
return aws(model.ProviderAWS, model.ServiceMemoryDB,
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
}
return model.Match{}, false
}
// awsResourceAfterRegion matches <resource>.<region>.<service>.amazonaws.com
// and rejects the bare <service>.<region>.amazonaws.com service endpoint.
func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) {
n := s.n
if n < 5 {
return model.Match{}, false
}
resource := s.label(0)
if resource == s.label(n-3) {
return model.Match{}, false
}
return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true
}
func classifyAWSS3(s *scanner) (model.Match, bool) {
n := s.n
// Access points: <ap>.s3-accesspoint[-fips][.dualstack].<region>.amazonaws.com
if n >= 5 {
svcIdx := -1
switch {
case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips":
svcIdx = n - 4
case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint":
svcIdx = n - 5
}
if svcIdx > 0 {
res := s.label(0)
if res != "s3-accesspoint" && res != "s3-accesspoint-fips" {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, res, s.label(n-3)), true
}
}
}
// Multi-region access point: <res>.mrap.accesspoint.s3-global.amazonaws.com
if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" &&
s.label(n-3) == "s3-global" {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), ""), true
}
// S3 Express: <bucket>.s3express-<zone>.<region>.amazonaws.com
if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
}
// S3 website: <bucket>.s3-website.<region>.amazonaws.com
if n >= 5 && s.label(n-4) == "s3-website" {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
}
// S3 website (region in the service label): <bucket>.s3-website-<region>.amazonaws.com
if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") {
return aws(model.ProviderAWS, model.ServiceS3,
model.PublicSemanticNone, s.label(0), ""), true
}
// Plain S3 without region: <bucket>.s3.amazonaws.com
if n == 4 && s.label(n-3) == "s3" {
res := s.label(0)
if res == "s3" {
return model.Match{}, false
}
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true
}
// Plain S3 with region: <bucket>.s3.<region>.amazonaws.com
if n >= 5 && s.label(n-4) == "s3" {
res := s.label(0)
if res == "s3" {
return model.Match{}, false
}
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
}
// Dualstack S3: <bucket>.s3.dualstack.<region>.amazonaws.com
if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" {
res := s.label(0)
if res == "s3" {
return model.Match{}, false
}
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
}
// Legacy regional form: <bucket>.s3-<region>.amazonaws.com (e.g. s3-eu-west-1)
if n == 4 {
svc := s.label(n - 3)
if region, ok := strings.CutPrefix(svc, "s3-"); ok && strings.Contains(region, "-") &&
!strings.HasPrefix(svc, "s3-website") && !strings.HasPrefix(svc, "s3-accesspoint") &&
!strings.HasPrefix(svc, "s3express-") {
res := s.label(0)
if res != "s3" {
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, region), true
}
}
}
return model.Match{}, false
}