karl/bucketeer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

provider capability model, on-demand scan/backfill, more providers

Browse Source 
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
  object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
  object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.

Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.

Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.

On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.

Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.

DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.

Verified live against real public resources:
  veteran1.nyc3.digitaloceanspaces.com            listable, entries
  bucket-01.s3-eu-west-1.amazonaws.com            listable, entries
  tartugeo-podcast.storage.googleapis.com         listable, entries
  eroticfeeldev.blob.core.windows.net/images      listable, entries
This commit is contained in:
StasonJatham committed 2026-09-22 14:47:49 +02:00
1 parent a366905a43
commit 62a7d2eb2e
23 files changed
+936 -165
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/probe/protocols.go
+24 -32
View File
@@ -18,7 +18,7 @@ import (
// database endpoint actually answers a read without credentials. No password,
// token or credential of any kind is ever sent, and no state is mutated.
func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureState, string, bool) {
func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (Result, bool) {
switch c.Service {
case model.ServiceRedis, model.ServiceElastiCache, model.ServiceMemoryDB:
return p.probeRedis(ctx, c.Hostname, dbPort(c.Service))
@@ -31,7 +31,7 @@ func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureS
case model.ServiceNeptune:
return p.probeHTTPReadable(ctx, "https://"+c.Hostname+"/status")
}
return model.ExposureUnknown, "", false
return Result{}, false
}
func dbPort(s model.Service) int {
@@ -75,7 +75,7 @@ func (p *Prober) dialTLS(ctx context.Context, host string, port int) (net.Conn,
}
// probeRedis sends PING. Redis answers +PONG when no authentication is required.
func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
func (p *Prober) probeRedis(ctx context.Context, host string, port int) (Result, bool) {
var lastErr error
for _, tlsOn := range []bool{false, true} {
var (
@@ -99,9 +99,9 @@ func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.E
resp := string(buf[:n])
switch {
case strings.HasPrefix(resp, "+PONG"):
return model.ExposurePublicRead, "redis PING answered without auth", true
return listRead("redis PING answered without auth"), true
case strings.Contains(resp, "NOAUTH"), strings.HasPrefix(resp, "-ERR"), strings.Contains(resp, "DENIED"), strings.Contains(resp, "protected mode"):
return model.ExposureAuthRequired, "redis requires authentication", true
return wrap(model.ExposureAuthRequired, "redis requires authentication"), true
}
// no/invalid response: try the other transport
}
@@ -109,17 +109,16 @@ func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.E
lastErr = fmt.Errorf("redis: no response")
}
st, detail := classifyError(lastErr)
return st, detail, true
return wrap(st, detail), true
}
// probePostgres negotiates TLS if offered, then sends a StartupMessage with no
// password. The server's AuthenticationRequest tells us whether it trusts the
// connection (AuthenticationOk) or requires a password.
func (p *Prober) probePostgres(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
func (p *Prober) probePostgres(ctx context.Context, host string, port int) (Result, bool) {
conn, err := p.dial(ctx, host, port)
if err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(p.cfg.Timeout))
@@ -148,14 +147,12 @@ func (p *Prober) probePostgres(ctx context.Context, host string, port int) (mode
binary.BigEndian.PutUint32(msg, uint32(4+len(payload)))
copy(msg[4:], payload)
if _, err := conn.Write(msg); err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
head := make([]byte, 5)
if _, err := io.ReadFull(conn, head); err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
kind := head[0]
length := int(binary.BigEndian.Uint32(head[1:5]))
@@ -165,20 +162,20 @@ func (p *Prober) probePostgres(ctx context.Context, host string, port int) (mode
if kind == 'R' && len(body) >= 4 {
authType := binary.BigEndian.Uint32(body[:4])
if authType == 0 {
return model.ExposurePublicRead, "postgres AuthenticationOk (trust, no password required)", true
return listRead("postgres AuthenticationOk (trust, no password required)"), true
}
return model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType), true
return wrap(model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType)), true
}
if kind == 'E' {
return model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)", true
return wrap(model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)"), true
}
}
return model.ExposureReachable, "postgres port open; auth state unclear", true
return wrap(model.ExposureReachable, "postgres port open; auth state unclear"), true
}
// probeMongo performs the OP_MSG handshake then a read-only listDatabases. A
// successful, non-empty reply means the deployment has no authentication.
func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
func (p *Prober) probeMongo(ctx context.Context, host string, port int) (Result, bool) {
var (
conn net.Conn
err error
@@ -188,8 +185,7 @@ func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.E
if err != nil {
conn, err = p.dial(ctx, host, port)
if err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
}
defer conn.Close()
@@ -197,33 +193,29 @@ func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.E
doc := bsonDoc{"listDatabases": int32(1), "$db": "admin"}
if err := writeOPMsg(conn, doc); err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
raw, err := readOPMsg(conn)
if err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
ok, errmsg := parseBSONFields(raw)
switch {
case ok == 1 && errmsg == "":
return model.ExposurePublicRead, "mongodb listDatabases succeeded without auth", true
return listRead("mongodb listDatabases succeeded without auth"), true
case strings.Contains(strings.ToLower(errmsg), "auth") || strings.Contains(strings.ToLower(errmsg), "unauthorized"):
return model.ExposureAuthRequired, "mongodb requires authentication", true
return wrap(model.ExposureAuthRequired, "mongodb requires authentication"), true
default:
return model.ExposureReachable, "mongodb replied but auth state unclear", true
return wrap(model.ExposureReachable, "mongodb replied but auth state unclear"), true
}
}
func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (model.ExposureState, string, bool) {
func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (Result, bool) {
status, _, err := p.do(ctx, endpoint)
if err != nil {
st, d := classifyError(err)
return st, d, true
return wrap(classifyError(err)), true
}
st, d := classifyStatus(status, "")
return st, d, true
return wrap(classifyStatus(status, "")), true
}
// --- minimal BSON/OP_MSG helpers (read-only) ---