Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
278 lines
5.7 KiB
Go
278 lines
5.7 KiB
Go
// Package model holds the compact enums and hot-path structs shared by the
|
|
// classifier, pipeline and store.
|
|
package model
|
|
|
|
import "time"
|
|
|
|
// Provider identifies a cloud/database/storage provider.
|
|
type Provider uint8
|
|
|
|
const (
|
|
ProviderUnknown Provider = iota
|
|
ProviderAWS
|
|
ProviderAzure
|
|
ProviderGCP
|
|
ProviderCloudflare
|
|
ProviderDigitalOcean
|
|
ProviderSupabase
|
|
ProviderOracle
|
|
)
|
|
|
|
func (p Provider) String() string {
|
|
switch p {
|
|
case ProviderAWS:
|
|
return "aws"
|
|
case ProviderAzure:
|
|
return "azure"
|
|
case ProviderGCP:
|
|
return "gcp"
|
|
case ProviderCloudflare:
|
|
return "cloudflare"
|
|
case ProviderDigitalOcean:
|
|
return "digitalocean"
|
|
case ProviderSupabase:
|
|
return "supabase"
|
|
case ProviderOracle:
|
|
return "oracle"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Service identifies the provider service the resource belongs to.
|
|
type Service uint8
|
|
|
|
const (
|
|
ServiceUnknown Service = iota
|
|
|
|
ServiceObjectStorage
|
|
ServiceS3
|
|
ServiceBlobStorage
|
|
ServiceDatabase
|
|
ServicePostgres
|
|
ServiceMySQL
|
|
ServiceRDS
|
|
ServiceDocumentDB
|
|
ServiceRedshift
|
|
ServiceOpenSearch
|
|
ServiceNeptune
|
|
ServiceRedis
|
|
ServiceMemoryDB
|
|
ServiceCosmosDB
|
|
ServiceFirebaseDB
|
|
ServiceElastiCache
|
|
ServiceSearch
|
|
ServiceQueue
|
|
ServiceFileStorage
|
|
ServiceTableStorage
|
|
ServiceWebApp
|
|
)
|
|
|
|
func (s Service) String() string {
|
|
switch s {
|
|
case ServiceObjectStorage:
|
|
return "object_storage"
|
|
case ServiceS3:
|
|
return "s3"
|
|
case ServiceBlobStorage:
|
|
return "blob_storage"
|
|
case ServiceDatabase:
|
|
return "database"
|
|
case ServicePostgres:
|
|
return "postgres"
|
|
case ServiceMySQL:
|
|
return "mysql"
|
|
case ServiceRDS:
|
|
return "rds"
|
|
case ServiceDocumentDB:
|
|
return "documentdb"
|
|
case ServiceRedshift:
|
|
return "redshift"
|
|
case ServiceOpenSearch:
|
|
return "opensearch"
|
|
case ServiceNeptune:
|
|
return "neptune"
|
|
case ServiceRedis:
|
|
return "redis"
|
|
case ServiceMemoryDB:
|
|
return "memorydb"
|
|
case ServiceCosmosDB:
|
|
return "cosmosdb"
|
|
case ServiceFirebaseDB:
|
|
return "firebase_db"
|
|
case ServiceElastiCache:
|
|
return "elasticache"
|
|
case ServiceSearch:
|
|
return "search"
|
|
case ServiceQueue:
|
|
return "queue"
|
|
case ServiceFileStorage:
|
|
return "file_storage"
|
|
case ServiceTableStorage:
|
|
return "table_storage"
|
|
case ServiceWebApp:
|
|
return "web_app"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Source is where a candidate was discovered.
|
|
type Source uint8
|
|
|
|
const (
|
|
SourceUnknown Source = iota
|
|
SourceDirectCT
|
|
SourceCNAME
|
|
SourceDNSAlias
|
|
SourceManual
|
|
)
|
|
|
|
func (s Source) String() string {
|
|
switch s {
|
|
case SourceDirectCT:
|
|
return "direct_ct"
|
|
case SourceCNAME:
|
|
return "cname"
|
|
case SourceDNSAlias:
|
|
return "dns_alias"
|
|
case SourceManual:
|
|
return "manual"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// PublicSemantic captures when the hostname format itself carries public
|
|
// accessibility meaning. It never asserts that data is actually exposed.
|
|
type PublicSemantic uint8
|
|
|
|
const (
|
|
PublicSemanticNone PublicSemantic = iota
|
|
PublicSemanticPublicEndpoint
|
|
PublicSemanticVPCEndpoint
|
|
PublicSemanticPublicURL
|
|
)
|
|
|
|
func (p PublicSemantic) String() string {
|
|
switch p {
|
|
case PublicSemanticPublicEndpoint:
|
|
return "public_endpoint"
|
|
case PublicSemanticVPCEndpoint:
|
|
return "vpc_endpoint"
|
|
case PublicSemanticPublicURL:
|
|
return "public_url"
|
|
default:
|
|
return "none"
|
|
}
|
|
}
|
|
|
|
// ExposureState is the outcome of an optional safe reachability check.
|
|
type ExposureState uint8
|
|
|
|
const (
|
|
ExposureUnknown ExposureState = iota
|
|
ExposurePublicDNS
|
|
ExposureReachable
|
|
ExposureAuthRequired
|
|
ExposurePublicRead
|
|
ExposurePrivate
|
|
ExposureNotFound
|
|
)
|
|
|
|
func (e ExposureState) String() string {
|
|
switch e {
|
|
case ExposurePublicDNS:
|
|
return "public_dns"
|
|
case ExposureReachable:
|
|
return "reachable"
|
|
case ExposureAuthRequired:
|
|
return "auth_required"
|
|
case ExposurePublicRead:
|
|
return "public_read"
|
|
case ExposurePrivate:
|
|
return "private"
|
|
case ExposureNotFound:
|
|
return "not_found"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// ParseProvider maps a stored provider string back to its enum.
|
|
func ParseProvider(s string) Provider {
|
|
switch s {
|
|
case "aws":
|
|
return ProviderAWS
|
|
case "azure":
|
|
return ProviderAzure
|
|
case "gcp":
|
|
return ProviderGCP
|
|
case "cloudflare":
|
|
return ProviderCloudflare
|
|
case "digitalocean":
|
|
return ProviderDigitalOcean
|
|
case "supabase":
|
|
return ProviderSupabase
|
|
case "oracle":
|
|
return ProviderOracle
|
|
default:
|
|
return ProviderUnknown
|
|
}
|
|
}
|
|
|
|
// ParseService maps a stored service string back to its enum.
|
|
func ParseService(s string) Service {
|
|
for svc := ServiceUnknown + 1; svc <= ServiceWebApp; svc++ {
|
|
if svc.String() == s {
|
|
return svc
|
|
}
|
|
}
|
|
return ServiceUnknown
|
|
}
|
|
|
|
// Match is the compact result of classifying a single hostname. The string
|
|
// fields are substrings of the input hostname (zero-copy).
|
|
type Match struct {
|
|
Provider Provider
|
|
Service Service
|
|
PublicSemantic PublicSemantic
|
|
ResourceSpecific bool
|
|
Region string
|
|
Resource string
|
|
}
|
|
|
|
// ProbeTarget is a persisted asset selected for re-probing/backfill.
|
|
type ProbeTarget struct {
|
|
Provider string
|
|
Service string
|
|
Hostname string
|
|
Resource string
|
|
Region string
|
|
}
|
|
|
|
// Entry is one indexed item of a readable resource (object key, container,
|
|
// prefix, database index, root key). It is metadata only — never the content.
|
|
type Entry struct {
|
|
Kind string // object | prefix | container | key | index
|
|
Name string
|
|
Size int64
|
|
Updated time.Time
|
|
ContentType string
|
|
ETag string
|
|
}
|
|
|
|
// Candidate is a provider resource discovered directly or through DNS.
|
|
type Candidate struct {
|
|
Hostname string // hostname that matched (provider hostname)
|
|
Original string // original CT hostname (equal to Hostname for direct matches)
|
|
Region string
|
|
Resource string
|
|
Container string // optional storage container/share hint (Azure)
|
|
Provider Provider
|
|
Service Service
|
|
Source Source
|
|
PublicSemantic PublicSemantic
|
|
ResourceSpecific bool
|
|
}
|