Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
161 lines
5.2 KiB
Go
161 lines
5.2 KiB
Go
package classify
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
)
|
|
|
|
func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match {
|
|
return model.Match{
|
|
Provider: p,
|
|
Service: s,
|
|
PublicSemantic: sem,
|
|
ResourceSpecific: true,
|
|
Region: region,
|
|
Resource: resource,
|
|
}
|
|
}
|
|
|
|
func classifyAWS(s *scanner) (model.Match, bool) {
|
|
n := s.n
|
|
if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {
|
|
return model.Match{}, false
|
|
}
|
|
if m, ok := classifyAWSS3(s); ok {
|
|
return m, true
|
|
}
|
|
|
|
// Service-after-region forms: <resource>.<region>.<service>.amazonaws.com
|
|
if n >= 5 {
|
|
switch s.label(n - 3) {
|
|
case "rds":
|
|
return awsResourceAfterRegion(s, model.ServiceRDS)
|
|
case "docdb":
|
|
return awsResourceAfterRegion(s, model.ServiceDocumentDB)
|
|
case "neptune":
|
|
return awsResourceAfterRegion(s, model.ServiceNeptune)
|
|
case "redshift", "redshift-serverless":
|
|
return awsResourceAfterRegion(s, model.ServiceRedshift)
|
|
case "es":
|
|
first := s.label(0)
|
|
switch {
|
|
case strings.HasPrefix(first, "search-"):
|
|
return aws(model.ProviderAWS, model.ServiceOpenSearch,
|
|
model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true
|
|
case strings.HasPrefix(first, "vpc-"):
|
|
return aws(model.ProviderAWS, model.ServiceOpenSearch,
|
|
model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true
|
|
}
|
|
case "cache":
|
|
return aws(model.ProviderAWS, model.ServiceElastiCache,
|
|
model.PublicSemanticNone, s.label(0), s.label(n-4)), true
|
|
}
|
|
}
|
|
|
|
// Service-before-region form: <resource>.memorydb.<region>.amazonaws.com
|
|
if n >= 5 && s.label(n-4) == "memorydb" {
|
|
return aws(model.ProviderAWS, model.ServiceMemoryDB,
|
|
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
|
|
}
|
|
return model.Match{}, false
|
|
}
|
|
|
|
// awsResourceAfterRegion matches <resource>.<region>.<service>.amazonaws.com
|
|
// and rejects the bare <service>.<region>.amazonaws.com service endpoint.
|
|
func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) {
|
|
n := s.n
|
|
if n < 5 {
|
|
return model.Match{}, false
|
|
}
|
|
resource := s.label(0)
|
|
if resource == s.label(n-3) {
|
|
return model.Match{}, false
|
|
}
|
|
return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true
|
|
}
|
|
|
|
func classifyAWSS3(s *scanner) (model.Match, bool) {
|
|
n := s.n
|
|
|
|
// Access points: <ap>.s3-accesspoint[-fips][.dualstack].<region>.amazonaws.com
|
|
if n >= 5 {
|
|
svcIdx := -1
|
|
switch {
|
|
case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips":
|
|
svcIdx = n - 4
|
|
case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint":
|
|
svcIdx = n - 5
|
|
}
|
|
if svcIdx > 0 {
|
|
res := s.label(0)
|
|
if res != "s3-accesspoint" && res != "s3-accesspoint-fips" {
|
|
return aws(model.ProviderAWS, model.ServiceS3,
|
|
model.PublicSemanticNone, res, s.label(n-3)), true
|
|
}
|
|
}
|
|
}
|
|
|
|
// Multi-region access point: <res>.mrap.accesspoint.s3-global.amazonaws.com
|
|
if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" &&
|
|
s.label(n-3) == "s3-global" {
|
|
return aws(model.ProviderAWS, model.ServiceS3,
|
|
model.PublicSemanticNone, s.label(0), ""), true
|
|
}
|
|
|
|
// S3 Express: <bucket>.s3express-<zone>.<region>.amazonaws.com
|
|
if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") {
|
|
return aws(model.ProviderAWS, model.ServiceS3,
|
|
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
|
|
}
|
|
|
|
// S3 website: <bucket>.s3-website.<region>.amazonaws.com
|
|
if n >= 5 && s.label(n-4) == "s3-website" {
|
|
return aws(model.ProviderAWS, model.ServiceS3,
|
|
model.PublicSemanticNone, s.label(0), s.label(n-3)), true
|
|
}
|
|
// S3 website (region in the service label): <bucket>.s3-website-<region>.amazonaws.com
|
|
if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") {
|
|
return aws(model.ProviderAWS, model.ServiceS3,
|
|
model.PublicSemanticNone, s.label(0), ""), true
|
|
}
|
|
|
|
// Plain S3 without region: <bucket>.s3.amazonaws.com
|
|
if n == 4 && s.label(n-3) == "s3" {
|
|
res := s.label(0)
|
|
if res == "s3" {
|
|
return model.Match{}, false
|
|
}
|
|
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true
|
|
}
|
|
// Plain S3 with region: <bucket>.s3.<region>.amazonaws.com
|
|
if n >= 5 && s.label(n-4) == "s3" {
|
|
res := s.label(0)
|
|
if res == "s3" {
|
|
return model.Match{}, false
|
|
}
|
|
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
|
|
}
|
|
// Dualstack S3: <bucket>.s3.dualstack.<region>.amazonaws.com
|
|
if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" {
|
|
res := s.label(0)
|
|
if res == "s3" {
|
|
return model.Match{}, false
|
|
}
|
|
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
|
|
}
|
|
// Legacy regional form: <bucket>.s3-<region>.amazonaws.com (e.g. s3-eu-west-1)
|
|
if n == 4 {
|
|
svc := s.label(n - 3)
|
|
if region, ok := strings.CutPrefix(svc, "s3-"); ok && strings.Contains(region, "-") &&
|
|
!strings.HasPrefix(svc, "s3-website") && !strings.HasPrefix(svc, "s3-accesspoint") &&
|
|
!strings.HasPrefix(svc, "s3express-") {
|
|
res := s.label(0)
|
|
if res != "s3" {
|
|
return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, region), true
|
|
}
|
|
}
|
|
}
|
|
return model.Match{}, false
|
|
}
|