// Package probe performs optional, minimal, non-destructive reachability and // readability checks. It only ever issues an unauthenticated HTTP request or an // unauthenticated TCP connect. It never sends credentials, downloads objects, // mutates anything, runs database queries, or attempts authentication. // // "readable" means an anonymous request was answered with data: for object // stores we use the vendor's anonymous list endpoint, for Firebase the REST // data endpoint, etc. For database endpoints there is no anonymous read concept, // so the best honest signal is whether the endpoint is publicly resolvable and // its port is open; we report that as reachable, never as readable. package probe import ( "context" "crypto/tls" "errors" "fmt" "io" mathrand "math/rand/v2" "net" "net/http" "net/url" "sync" "time" "tea.karlcom.de/karl/bucketeer/internal/metrics" "tea.karlcom.de/karl/bucketeer/internal/model" ) // UpdateFunc records the observed exposure state and a short factual detail. type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState, detail string) type Config struct { Workers int Timeout time.Duration Queue int Metrics *metrics.Metrics Update UpdateFunc } type Prober struct { cfg Config sample *sampleQueue client *http.Client wg sync.WaitGroup } // sampleQueue is a memory-bounded queue that keeps a random sample of recent // submissions rather than a FIFO. With FIFO, one high-volume hostname template // (e.g. *.mrap.accesspoint.s3-global.amazonaws.com) fills the entire queue and // no other asset type is ever probed. Random replacement means every template // gets sampled over time. It is not on the certificate hot path. type sampleQueue struct { mu sync.Mutex items []model.Candidate cap int notify chan struct{} } func newSampleQueue(capacity int) *sampleQueue { return &sampleQueue{cap: capacity, notify: make(chan struct{}, 1)} } func (q *sampleQueue) push(c model.Candidate) { q.mu.Lock() if len(q.items) < q.cap { q.items = append(q.items, c) } else { q.items[mathrand.IntN(len(q.items))] = c } q.mu.Unlock() select { case q.notify <- struct{}{}: default: } } func (q *sampleQueue) pop() (model.Candidate, bool) { q.mu.Lock() n := len(q.items) if n == 0 { q.mu.Unlock() return model.Candidate{}, false } i := mathrand.IntN(n) c := q.items[i] q.items[i] = q.items[n-1] q.items = q.items[:n-1] q.mu.Unlock() return c, true } func New(cfg Config) *Prober { if cfg.Workers <= 0 { cfg.Workers = 4 } if cfg.Timeout <= 0 { cfg.Timeout = 5 * time.Second } if cfg.Queue <= 0 { cfg.Queue = 1024 } return &Prober{ cfg: cfg, sample: newSampleQueue(cfg.Queue), client: &http.Client{ Timeout: cfg.Timeout, // Observe the first response; do not follow redirects so region // hints (e.g. S3 301 PermanentRedirect) stay visible. CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // observation only, no trust decision MaxIdleConns: cfg.Workers * 2, MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second, DialContext: (&net.Dialer{ Timeout: cfg.Timeout, KeepAlive: 30 * time.Second, }).DialContext, }, }, } } func (p *Prober) Start(ctx context.Context) { for i := 0; i < p.cfg.Workers; i++ { p.wg.Add(1) go p.worker(ctx) } } // Submit enqueues a candidate without blocking. When the sample is full the // oldest-slot candidate is replaced at random; this is load shedding, not an // error, so it is not counted as a drop. func (p *Prober) Submit(c model.Candidate) { if p.cfg.Metrics != nil { p.cfg.Metrics.ProbeAttempts.Add(1) } p.sample.push(c) } func (p *Prober) Wait() { p.wg.Wait() } func (p *Prober) worker(ctx context.Context) { defer p.wg.Done() for { select { case <-ctx.Done(): return case <-p.sample.notify: } for { c, ok := p.sample.pop() if !ok { break } state, detail := p.Probe(ctx, c) if p.cfg.Metrics != nil && state == model.ExposurePublicRead { p.cfg.Metrics.ProbePublic.Add(1) } if p.cfg.Update != nil { p.cfg.Update(ctx, c, state, detail) } } } } // Probe runs the vendor-appropriate check for a single candidate. func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) { switch c.Provider { case model.ProviderAWS: switch c.Service { case model.ServiceS3: return p.checkS3(ctx, c) case model.ServiceRDS, model.ServiceDocumentDB: return p.checkTCP(ctx, c.Hostname, 5432) case model.ServiceNeptune: return p.checkTCP(ctx, c.Hostname, 8182) case model.ServiceElastiCache, model.ServiceMemoryDB: return p.checkTCP(ctx, c.Hostname, 6379) case model.ServiceRedshift: return p.checkTCP(ctx, c.Hostname, 5439) case model.ServiceOpenSearch: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } case model.ProviderAzure: switch c.Service { case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage: // Anonymous container/file/share/queue/table listing. return p.checkHTTP(ctx, "https://"+c.Hostname+"/?comp=list") case model.ServiceWebApp, model.ServiceSearch: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") case model.ServiceCosmosDB: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") case model.ServiceDatabase: // Azure SQL return p.checkTCP(ctx, c.Hostname, 1433) case model.ServicePostgres: return p.checkTCP(ctx, c.Hostname, 5432) case model.ServiceMySQL: return p.checkTCP(ctx, c.Hostname, 3306) case model.ServiceRedis: return p.checkTCP(ctx, c.Hostname, 6380) } case model.ProviderGCP: switch c.Service { case model.ServiceObjectStorage: return p.checkGCS(ctx, c) case model.ServiceFirebaseDB: return p.checkFirebase(ctx, c) case model.ServiceWebApp: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } case model.ProviderCloudflare: return p.checkS3(ctx, c) // R2 is S3-compatible (and r2.dev serves objects) case model.ProviderDigitalOcean: if c.Service == model.ServiceObjectStorage { return p.checkS3(ctx, c) } return p.checkTCP(ctx, c.Hostname, 5432) case model.ProviderSupabase: if c.Service == model.ServicePostgres { return p.checkTCP(ctx, c.Hostname, 5432) } return p.checkHTTP(ctx, "https://"+c.Hostname+"/rest/v1/") case model.ProviderOracle: return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } return p.checkHTTP(ctx, "https://"+c.Hostname+"/") } // checkS3 queries the S3 ListObjectsV2 anonymous endpoint. A 200 means the // bucket allows anonymous listing (public read); 403 means it exists but is not // public; 404 means no such bucket. 301/307 carries the real region. func (p *Prober) checkS3(ctx context.Context, c model.Candidate) (model.ExposureState, string) { status, header, err := p.do(ctx, "https://"+c.Hostname+"/?list-type=2&max-keys=1") if err != nil { return classifyError(err) } if status == http.StatusMovedPermanently || status == http.StatusTemporaryRedirect || status == http.StatusPermanentRedirect { if region := header.Get("x-amz-bucket-region"); region != "" { return model.ExposureReachable, "s3 redirect region=" + region } return model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status) } return classifyStatus(status, "s3") } // checkGCS uses the public JSON API to test object listing, which requires the // bucket to grant allUsers read access. func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.ExposureState, string) { bucket := c.Resource if bucket == "" { bucket = c.Hostname } api := "https://storage.googleapis.com/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1" status, _, err := p.do(ctx, api) if err != nil { return classifyError(err) } return classifyStatus(status, "gcs") } // checkFirebase queries the Realtime Database REST endpoint with shallow=true so // only keys are returned. 200 means the security rules allow anonymous reads. func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) (model.ExposureState, string) { status, _, err := p.do(ctx, "https://"+c.Hostname+"/.json?shallow=true") if err != nil { return classifyError(err) } return classifyStatus(status, "firebase") } func (p *Prober) checkHTTP(ctx context.Context, endpoint string) (model.ExposureState, string) { status, _, err := p.do(ctx, endpoint) if err != nil { return classifyError(err) } return classifyStatus(status, "") } // checkTCP reports whether a database endpoint is publicly resolvable and its // port is open. It intentionally does not attempt any protocol handshake or // authentication, so it can never be reported as readable. func (p *Prober) checkTCP(ctx context.Context, host string, port int) (model.ExposureState, string) { d := net.Dialer{Timeout: p.cfg.Timeout} conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, fmt.Sprint(port))) if err != nil { return classifyError(err) } _ = conn.Close() return model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port) } // do issues a single GET and reads at most a tiny prefix of the body before // closing it, so no object is ever downloaded. func (p *Prober) do(ctx context.Context, endpoint string) (int, http.Header, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return 0, nil, err } resp, err := p.client.Do(req) if err != nil { return 0, nil, err } defer resp.Body.Close() _, _ = io.CopyN(io.Discard, resp.Body, 512) return resp.StatusCode, resp.Header, nil } func classifyStatus(status int, tag string) (model.ExposureState, string) { prefix := "" if tag != "" { prefix = tag + " " } switch { case status >= 200 && status < 300: return model.ExposurePublicRead, fmt.Sprintf("%sstatus=%d anonymous listing allowed", prefix, status) case status == http.StatusUnauthorized || status == http.StatusForbidden: return model.ExposureAuthRequired, fmt.Sprintf("%sstatus=%d authentication required", prefix, status) case status == http.StatusNotFound || status == http.StatusGone: return model.ExposureNotFound, fmt.Sprintf("%sstatus=%d not found", prefix, status) default: return model.ExposureReachable, fmt.Sprintf("%sstatus=%d", prefix, status) } } func classifyError(err error) (model.ExposureState, string) { var dnsErr *net.DNSError if errors.As(err, &dnsErr) { if dnsErr.IsNotFound { return model.ExposurePrivate, "dns: no public record" } return model.ExposureUnknown, "dns: " + dnsErr.Err } var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { return model.ExposureUnknown, "timeout" } return model.ExposureUnknown, err.Error() } // Describe is a helper for logs/tests. func Describe(s model.ExposureState) string { return fmt.Sprintf("exposure=%s", s) }