// Package probe performs optional, minimal, non-destructive reachability and // readability checks. It only ever issues an unauthenticated HTTP request or an // unauthenticated TCP connect. It never sends credentials, downloads objects, // mutates anything, runs database queries, or attempts authentication. // // Readability is reported as separate capabilities, because they differ by // provider: // // exists the hostname resolves / the service answers at all // object_readable an anonymous GET of a known object/key succeeds // listable the provider's anonymous list API returns the object names // // Only a listable resource can be enumerated and indexed. Azure grants access // per container ("Blob" = object read only, "Container" = listable), and // Cloudflare R2 public URLs serve objects but never allow anonymous listing. package probe import ( "context" "crypto/tls" "errors" "fmt" "io" mathrand "math/rand/v2" "net" "net/http" "net/url" "strings" "sync" "time" "tea.karlcom.de/karl/bucketeer/internal/index" "tea.karlcom.de/karl/bucketeer/internal/metrics" "tea.karlcom.de/karl/bucketeer/internal/model" ) // Result is the outcome of probing one candidate. type Result struct { Exposure model.ExposureState Detail string Listable bool ObjectReadable bool } func res(state model.ExposureState, detail string) Result { return Result{Exposure: state, Detail: detail} } // listRead marks a resource whose names can be enumerated anonymously. func listRead(detail string) Result { return Result{Exposure: model.ExposurePublicRead, Detail: detail, Listable: true, ObjectReadable: true} } // objRead marks a resource with a publicly readable object but no listing. func objRead(detail string) Result { return Result{Exposure: model.ExposurePublicRead, Detail: detail, ObjectReadable: true} } // UpdateFunc records the observed capabilities and a short factual detail. type UpdateFunc func(ctx context.Context, c model.Candidate, r Result) type Config struct { Workers int Timeout time.Duration Queue int Deep bool // unauthenticated protocol reads for database services Metrics *metrics.Metrics Update UpdateFunc // Indexer, when set, lists metadata for listable resources and hands the // entries to OnIndex. Contents are never downloaded. Indexer *index.Indexer OnIndex func(ctx context.Context, c model.Candidate, entries []model.Entry) } type Prober struct { cfg Config sample *sampleQueue client *http.Client stop chan struct{} wg sync.WaitGroup } // sampleQueue is a memory-bounded queue that keeps a random sample of recent // submissions rather than a FIFO, so one high-volume hostname template cannot // starve every other asset type. It is not on the certificate hot path. type sampleQueue struct { mu sync.Mutex items []model.Candidate cap int notify chan struct{} } func newSampleQueue(capacity int) *sampleQueue { return &sampleQueue{cap: capacity, notify: make(chan struct{}, 1)} } func (q *sampleQueue) push(c model.Candidate) { q.mu.Lock() if len(q.items) < q.cap { q.items = append(q.items, c) } else { q.items[mathrand.IntN(len(q.items))] = c } q.mu.Unlock() select { case q.notify <- struct{}{}: default: } } func (q *sampleQueue) len() int { q.mu.Lock() defer q.mu.Unlock() return len(q.items) } func (q *sampleQueue) pop() (model.Candidate, bool) { q.mu.Lock() n := len(q.items) if n == 0 { q.mu.Unlock() return model.Candidate{}, false } i := mathrand.IntN(n) c := q.items[i] q.items[i] = q.items[n-1] q.items = q.items[:n-1] q.mu.Unlock() return c, true } func New(cfg Config) *Prober { if cfg.Workers <= 0 { cfg.Workers = 4 } if cfg.Timeout <= 0 { cfg.Timeout = 5 * time.Second } if cfg.Queue <= 0 { cfg.Queue = 1024 } return &Prober{ cfg: cfg, sample: newSampleQueue(cfg.Queue), stop: make(chan struct{}), client: &http.Client{ Timeout: cfg.Timeout, // Observe the first response; do not follow redirects so region // hints (e.g. S3 301 PermanentRedirect) stay visible. CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // observation only, no trust decision MaxIdleConns: cfg.Workers * 2, MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second, DialContext: (&net.Dialer{ Timeout: cfg.Timeout, KeepAlive: 30 * time.Second, }).DialContext, }, }, } } func (p *Prober) Start(ctx context.Context) { for i := 0; i < p.cfg.Workers; i++ { p.wg.Add(1) go p.worker(ctx) } } // Submit enqueues a candidate without blocking. When the sample is full the // oldest-slot candidate is replaced at random: load shedding, not an error. func (p *Prober) Submit(c model.Candidate) { if p.cfg.Metrics != nil { p.cfg.Metrics.ProbeAttempts.Add(1) } p.sample.push(c) } // Close stops the workers after any in-flight probe. It must be called before // Wait, otherwise Wait blocks until ctx is cancelled. func (p *Prober) Close() { close(p.stop) } func (p *Prober) Wait() { p.wg.Wait() } // QueueLen reports the current probe sample depth. func (p *Prober) QueueLen() int { return p.sample.len() } func (p *Prober) worker(ctx context.Context) { defer p.wg.Done() for { select { case <-ctx.Done(): return case <-p.stop: return case <-p.sample.notify: } for { c, ok := p.sample.pop() if !ok { break } r := p.ProbeDetailed(ctx, c) if p.cfg.Metrics != nil && r.Exposure == model.ExposurePublicRead { p.cfg.Metrics.ProbePublic.Add(1) } if p.cfg.Update != nil { p.cfg.Update(ctx, c, r) } // Only listable resources can be enumerated. if r.Listable && p.cfg.Indexer != nil && p.cfg.OnIndex != nil { if entries, err := p.cfg.Indexer.List(ctx, c); err == nil && len(entries) > 0 { p.cfg.OnIndex(ctx, c, entries) if p.cfg.Metrics != nil { p.cfg.Metrics.EntriesIndexed.Add(uint64(len(entries))) } } } } } } // Probe is the (exposure, detail) convenience view. func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) { r := p.ProbeDetailed(ctx, c) return r.Exposure, r.Detail } // ProbeDetailed runs the vendor-appropriate check and reports capabilities. func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result { // Deep, read-only protocol checks take precedence for database services. if p.cfg.Deep { if r, ok := p.deepDB(ctx, c); ok { return r } } switch c.Provider { case model.ProviderAWS: switch c.Service { case model.ServiceS3: return p.checkS3(ctx, c) case model.ServiceRDS, model.ServiceDocumentDB: return p.checkTCP(ctx, c.Hostname, 5432) case model.ServiceNeptune: return p.checkHTTP(ctx, "https://"+c.Hostname+"/status") case model.ServiceElastiCache, model.ServiceMemoryDB: return p.checkTCP(ctx, c.Hostname, 6379) case model.ServiceRedshift: return p.checkTCP(ctx, c.Hostname, 5439) case model.ServiceOpenSearch: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } case model.ProviderAzure: switch c.Service { case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage: return p.checkAzureStorage(ctx, c) case model.ServiceWebApp, model.ServiceSearch, model.ServiceCosmosDB: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") case model.ServiceDatabase: // Azure SQL return p.checkTCP(ctx, c.Hostname, 1433) case model.ServicePostgres: return p.checkTCP(ctx, c.Hostname, 5432) case model.ServiceMySQL: return p.checkTCP(ctx, c.Hostname, 3306) case model.ServiceRedis: return p.checkTCP(ctx, c.Hostname, 6380) } case model.ProviderGCP: switch c.Service { case model.ServiceObjectStorage: return p.checkGCS(ctx, c) case model.ServiceFirebaseDB: return p.checkFirebase(ctx, c) case model.ServiceWebApp: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } case model.ProviderCloudflare: // R2's authenticated S3 API is not anonymous; public URLs (r2.dev or a // custom domain) serve objects but explicitly do not allow listing. if strings.HasSuffix(c.Hostname, ".r2.dev") { return res(model.ExposureReachable, "r2.dev public object host; anonymous listing is not supported") } return p.checkS3(ctx, c) case model.ProviderDigitalOcean: if c.Service == model.ServiceObjectStorage { return p.checkS3(ctx, c) } return p.checkTCP(ctx, c.Hostname, 5432) case model.ProviderSupabase: if c.Service == model.ServicePostgres { return p.checkTCP(ctx, c.Hostname, 5432) } return p.checkHTTP(ctx, "https://"+c.Hostname+"/rest/v1/") case model.ProviderOracle: return p.checkS3(ctx, c) // S3-compatible } return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } // checkAzureStorage: account-level listing is disabled anonymously (403/404). // Anonymous access is per container ("Blob" = object read, "Container" = // listable), and the container name is not in the hostname, so probe a bounded // set of common names, or the known one when a URL path provided it. func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) Result { containers := index.AzureContainers(c.Resource) known := c.Container != "" if known { containers = []string{c.Container} } for _, name := range containers { status, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"?restype=container&comp=list") if err != nil { continue } if status >= 200 && status < 300 { return listRead("azure container '" + name + "' is publicly listable") } // "Blob" mode: object readable, container not listable. for _, key := range []string{"index.html", "robots.txt"} { if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"/"+key); err == nil && s >= 200 && s < 300 { return objRead("azure container '" + name + "' objects readable but not listable") } } } if status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list"); err == nil && status >= 200 && status < 300 { return listRead("azure account-level anonymous listing succeeded") } return res(model.ExposureReachable, "azure account listing requires auth; no public container found") } // checkS3 queries ListObjectsV2 anonymously. 200 = listable; 403 with a readable // known object = object readable only; 404 = no such bucket; 301/307 = region. func (p *Prober) checkS3(ctx context.Context, c model.Candidate) Result { status, header, err := p.do(ctx, "https://"+c.Hostname+"/?list-type=2&max-keys=1") if err != nil { return wrap(classifyError(err)) } if status == http.StatusMovedPermanently || status == http.StatusTemporaryRedirect || status == http.StatusPermanentRedirect { if region := header.Get("x-amz-bucket-region"); region != "" { return res(model.ExposureReachable, "s3 redirect region="+region) } return res(model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status)) } if status >= 200 && status < 300 { return listRead(fmt.Sprintf("s3 status=%d anonymous listing allowed", status)) } if status == http.StatusForbidden || status == http.StatusUnauthorized { for _, key := range []string{"index.html", "robots.txt"} { if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+key); err == nil && s >= 200 && s < 300 { return objRead("s3 object readable (" + key + ") although listing is denied") } } } return wrap(classifyStatus(status, "s3")) } // checkGCS uses the public JSON API object list (requires storage.objects.list). func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) Result { bucket := c.Resource if bucket == "" { bucket = c.Hostname } api := "https://storage.googleapis.com/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1" status, _, err := p.do(ctx, api) if err != nil { return wrap(classifyError(err)) } if status >= 200 && status < 300 { return listRead(fmt.Sprintf("gcs status=%d anonymous listing allowed", status)) } if status == http.StatusForbidden || status == http.StatusUnauthorized { for _, key := range []string{"index.html", "robots.txt"} { obj := "https://storage.googleapis.com/" + url.PathEscape(bucket) + "/" + key if s, _, err := p.do(ctx, obj); err == nil && s >= 200 && s < 300 { return objRead("gcs object readable (" + key + ") although listing is denied") } } } return wrap(classifyStatus(status, "gcs")) } // checkFirebase queries the Realtime Database REST endpoint with shallow=true. func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) Result { status, _, err := p.do(ctx, "https://"+c.Hostname+"/.json?shallow=true") if err != nil { return wrap(classifyError(err)) } if status >= 200 && status < 300 { return listRead("firebase rules allow anonymous read") } return wrap(classifyStatus(status, "firebase")) } func (p *Prober) checkHTTP(ctx context.Context, endpoint string) Result { status, _, err := p.do(ctx, endpoint) if err != nil { return wrap(classifyError(err)) } return wrap(classifyStatus(status, "")) } // checkTCP reports whether a database endpoint is publicly resolvable and its // port is open. It never authenticates, so it can never be readable. func (p *Prober) checkTCP(ctx context.Context, host string, port int) Result { d := net.Dialer{Timeout: p.cfg.Timeout} conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, fmt.Sprint(port))) if err != nil { return wrap(classifyError(err)) } _ = conn.Close() return res(model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port)) } // do issues a single GET and reads at most a tiny prefix of the body before // closing it, so no object is ever downloaded. func (p *Prober) do(ctx context.Context, endpoint string) (int, http.Header, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return 0, nil, err } resp, err := p.client.Do(req) if err != nil { return 0, nil, err } defer resp.Body.Close() _, _ = io.CopyN(io.Discard, resp.Body, 512) return resp.StatusCode, resp.Header, nil } func wrap(state model.ExposureState, detail string) Result { return Result{Exposure: state, Detail: detail} } func classifyStatus(status int, tag string) (model.ExposureState, string) { prefix := "" if tag != "" { prefix = tag + " " } switch { case status >= 200 && status < 300: // Generic HTTP check: anonymous access succeeded, but this is not a // storage list API, so it does not imply the resource is listable. return model.ExposurePublicRead, fmt.Sprintf("%sstatus=%d anonymous access succeeded", prefix, status) case status == http.StatusUnauthorized || status == http.StatusForbidden: return model.ExposureAuthRequired, fmt.Sprintf("%sstatus=%d authentication required", prefix, status) case status == http.StatusNotFound || status == http.StatusGone: return model.ExposureNotFound, fmt.Sprintf("%sstatus=%d not found", prefix, status) default: return model.ExposureReachable, fmt.Sprintf("%sstatus=%d", prefix, status) } } func classifyError(err error) (model.ExposureState, string) { var dnsErr *net.DNSError if errors.As(err, &dnsErr) { if dnsErr.IsNotFound { return model.ExposurePrivate, "dns: no public record" } return model.ExposureUnknown, "dns: " + dnsErr.Err } var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { return model.ExposureUnknown, "timeout" } return model.ExposureUnknown, err.Error() } // Describe is a helper for logs/tests. func Describe(s model.ExposureState) string { return fmt.Sprintf("exposure=%s", s) }