provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
This commit is contained in:
1 parent
a366905a43
commit
62a7d2eb2e
23 files changed
+936
-165
No files matched your search
+141
-73
@@ -3,11 +3,16 @@
|
||||
// unauthenticated TCP connect. It never sends credentials, downloads objects,
|
||||
// mutates anything, runs database queries, or attempts authentication.
|
||||
//
|
||||
// "readable" means an anonymous request was answered with data: for object
|
||||
// stores we use the vendor's anonymous list endpoint, for Firebase the REST
|
||||
// data endpoint, etc. For database endpoints there is no anonymous read concept,
|
||||
// so the best honest signal is whether the endpoint is publicly resolvable and
|
||||
// its port is open; we report that as reachable, never as readable.
|
||||
// Readability is reported as separate capabilities, because they differ by
|
||||
// provider:
|
||||
//
|
||||
// exists the hostname resolves / the service answers at all
|
||||
// object_readable an anonymous GET of a known object/key succeeds
|
||||
// listable the provider's anonymous list API returns the object names
|
||||
//
|
||||
// Only a listable resource can be enumerated and indexed. Azure grants access
|
||||
// per container ("Blob" = object read only, "Container" = listable), and
|
||||
// Cloudflare R2 public URLs serve objects but never allow anonymous listing.
|
||||
package probe
|
||||
|
||||
import (
|
||||
@@ -20,6 +25,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -28,8 +34,30 @@ import (
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
// UpdateFunc records the observed exposure state and a short factual detail.
|
||||
type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState, detail string)
|
||||
// Result is the outcome of probing one candidate.
|
||||
type Result struct {
|
||||
Exposure model.ExposureState
|
||||
Detail string
|
||||
Listable bool
|
||||
ObjectReadable bool
|
||||
}
|
||||
|
||||
func res(state model.ExposureState, detail string) Result {
|
||||
return Result{Exposure: state, Detail: detail}
|
||||
}
|
||||
|
||||
// listRead marks a resource whose names can be enumerated anonymously.
|
||||
func listRead(detail string) Result {
|
||||
return Result{Exposure: model.ExposurePublicRead, Detail: detail, Listable: true, ObjectReadable: true}
|
||||
}
|
||||
|
||||
// objRead marks a resource with a publicly readable object but no listing.
|
||||
func objRead(detail string) Result {
|
||||
return Result{Exposure: model.ExposurePublicRead, Detail: detail, ObjectReadable: true}
|
||||
}
|
||||
|
||||
// UpdateFunc records the observed capabilities and a short factual detail.
|
||||
type UpdateFunc func(ctx context.Context, c model.Candidate, r Result)
|
||||
|
||||
type Config struct {
|
||||
Workers int
|
||||
@@ -39,8 +67,8 @@ type Config struct {
|
||||
Metrics *metrics.Metrics
|
||||
Update UpdateFunc
|
||||
|
||||
// Indexer, when set, lists metadata for resources confirmed public_read and
|
||||
// hands the entries to OnIndex. Contents are never downloaded.
|
||||
// Indexer, when set, lists metadata for listable resources and hands the
|
||||
// entries to OnIndex. Contents are never downloaded.
|
||||
Indexer *index.Indexer
|
||||
OnIndex func(ctx context.Context, c model.Candidate, entries []model.Entry)
|
||||
}
|
||||
@@ -54,10 +82,8 @@ type Prober struct {
|
||||
}
|
||||
|
||||
// sampleQueue is a memory-bounded queue that keeps a random sample of recent
|
||||
// submissions rather than a FIFO. With FIFO, one high-volume hostname template
|
||||
// (e.g. *.mrap.accesspoint.s3-global.amazonaws.com) fills the entire queue and
|
||||
// no other asset type is ever probed. Random replacement means every template
|
||||
// gets sampled over time. It is not on the certificate hot path.
|
||||
// submissions rather than a FIFO, so one high-volume hostname template cannot
|
||||
// starve every other asset type. It is not on the certificate hot path.
|
||||
type sampleQueue struct {
|
||||
mu sync.Mutex
|
||||
items []model.Candidate
|
||||
@@ -147,8 +173,7 @@ func (p *Prober) Start(ctx context.Context) {
|
||||
}
|
||||
|
||||
// Submit enqueues a candidate without blocking. When the sample is full the
|
||||
// oldest-slot candidate is replaced at random; this is load shedding, not an
|
||||
// error, so it is not counted as a drop.
|
||||
// oldest-slot candidate is replaced at random: load shedding, not an error.
|
||||
func (p *Prober) Submit(c model.Candidate) {
|
||||
if p.cfg.Metrics != nil {
|
||||
p.cfg.Metrics.ProbeAttempts.Add(1)
|
||||
@@ -180,14 +205,15 @@ func (p *Prober) worker(ctx context.Context) {
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
state, detail := p.Probe(ctx, c)
|
||||
if p.cfg.Metrics != nil && state == model.ExposurePublicRead {
|
||||
r := p.ProbeDetailed(ctx, c)
|
||||
if p.cfg.Metrics != nil && r.Exposure == model.ExposurePublicRead {
|
||||
p.cfg.Metrics.ProbePublic.Add(1)
|
||||
}
|
||||
if p.cfg.Update != nil {
|
||||
p.cfg.Update(ctx, c, state, detail)
|
||||
p.cfg.Update(ctx, c, r)
|
||||
}
|
||||
if state == model.ExposurePublicRead && p.cfg.Indexer != nil && p.cfg.OnIndex != nil {
|
||||
// Only listable resources can be enumerated.
|
||||
if r.Listable && p.cfg.Indexer != nil && p.cfg.OnIndex != nil {
|
||||
if entries, err := p.cfg.Indexer.List(ctx, c); err == nil && len(entries) > 0 {
|
||||
p.cfg.OnIndex(ctx, c, entries)
|
||||
if p.cfg.Metrics != nil {
|
||||
@@ -199,13 +225,18 @@ func (p *Prober) worker(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Probe runs the vendor-appropriate check for a single candidate.
|
||||
// Probe is the (exposure, detail) convenience view.
|
||||
func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
// Deep, read-only protocol checks take precedence for database services and
|
||||
// can distinguish "reachable" from "actually readable without credentials".
|
||||
r := p.ProbeDetailed(ctx, c)
|
||||
return r.Exposure, r.Detail
|
||||
}
|
||||
|
||||
// ProbeDetailed runs the vendor-appropriate check and reports capabilities.
|
||||
func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result {
|
||||
// Deep, read-only protocol checks take precedence for database services.
|
||||
if p.cfg.Deep {
|
||||
if state, detail, ok := p.deepDB(ctx, c); ok {
|
||||
return state, detail
|
||||
if r, ok := p.deepDB(ctx, c); ok {
|
||||
return r
|
||||
}
|
||||
}
|
||||
switch c.Provider {
|
||||
@@ -216,7 +247,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
|
||||
case model.ServiceRDS, model.ServiceDocumentDB:
|
||||
return p.checkTCP(ctx, c.Hostname, 5432)
|
||||
case model.ServiceNeptune:
|
||||
return p.checkTCP(ctx, c.Hostname, 8182)
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/status")
|
||||
case model.ServiceElastiCache, model.ServiceMemoryDB:
|
||||
return p.checkTCP(ctx, c.Hostname, 6379)
|
||||
case model.ServiceRedshift:
|
||||
@@ -228,9 +259,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
|
||||
switch c.Service {
|
||||
case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
|
||||
return p.checkAzureStorage(ctx, c)
|
||||
case model.ServiceWebApp, model.ServiceSearch:
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
case model.ServiceCosmosDB:
|
||||
case model.ServiceWebApp, model.ServiceSearch, model.ServiceCosmosDB:
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
case model.ServiceDatabase: // Azure SQL
|
||||
return p.checkTCP(ctx, c.Hostname, 1433)
|
||||
@@ -251,7 +280,12 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
}
|
||||
case model.ProviderCloudflare:
|
||||
return p.checkS3(ctx, c) // R2 is S3-compatible (and r2.dev serves objects)
|
||||
// R2's authenticated S3 API is not anonymous; public URLs (r2.dev or a
|
||||
// custom domain) serve objects but explicitly do not allow listing.
|
||||
if strings.HasSuffix(c.Hostname, ".r2.dev") {
|
||||
return res(model.ExposureReachable, "r2.dev public object host; anonymous listing is not supported")
|
||||
}
|
||||
return p.checkS3(ctx, c)
|
||||
case model.ProviderDigitalOcean:
|
||||
if c.Service == model.ServiceObjectStorage {
|
||||
return p.checkS3(ctx, c)
|
||||
@@ -263,52 +297,70 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
|
||||
}
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/rest/v1/")
|
||||
case model.ProviderOracle:
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
return p.checkS3(ctx, c) // S3-compatible
|
||||
}
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
}
|
||||
|
||||
// checkAzureStorage probes the account-level container list. Anonymous read is
|
||||
// configured per container in Azure ("Public access level"), never at the
|
||||
// account level, so a 401/403 here means only that account listing needs auth —
|
||||
// it does NOT mean the containers are unreadable. We therefore report reachable
|
||||
// rather than auth_required, and only index if account listing is unexpectedly
|
||||
// public (which would let us discover container names).
|
||||
func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list")
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
// checkAzureStorage: account-level listing is disabled anonymously (403/404).
|
||||
// Anonymous access is per container ("Blob" = object read, "Container" =
|
||||
// listable), and the container name is not in the hostname, so probe a bounded
|
||||
// set of common names, or the known one when a URL path provided it.
|
||||
func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) Result {
|
||||
containers := index.AzureContainers(c.Resource)
|
||||
known := c.Container != ""
|
||||
if known {
|
||||
containers = []string{c.Container}
|
||||
}
|
||||
switch {
|
||||
case status >= 200 && status < 300:
|
||||
return model.ExposurePublicRead, "azure account-level anonymous listing succeeded"
|
||||
case status == http.StatusUnauthorized || status == http.StatusForbidden:
|
||||
return model.ExposureReachable, "azure account listing requires auth; per-container readability needs a container name"
|
||||
default:
|
||||
return classifyStatus(status, "azure")
|
||||
for _, name := range containers {
|
||||
status, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"?restype=container&comp=list")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if status >= 200 && status < 300 {
|
||||
return listRead("azure container '" + name + "' is publicly listable")
|
||||
}
|
||||
// "Blob" mode: object readable, container not listable.
|
||||
for _, key := range []string{"index.html", "robots.txt"} {
|
||||
if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+name+"/"+key); err == nil && s >= 200 && s < 300 {
|
||||
return objRead("azure container '" + name + "' objects readable but not listable")
|
||||
}
|
||||
}
|
||||
}
|
||||
if status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list"); err == nil && status >= 200 && status < 300 {
|
||||
return listRead("azure account-level anonymous listing succeeded")
|
||||
}
|
||||
return res(model.ExposureReachable, "azure account listing requires auth; no public container found")
|
||||
}
|
||||
|
||||
// checkS3 queries the S3 ListObjectsV2 anonymous endpoint. A 200 means the
|
||||
// bucket allows anonymous listing (public read); 403 means it exists but is not
|
||||
// public; 404 means no such bucket. 301/307 carries the real region.
|
||||
func (p *Prober) checkS3(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
// checkS3 queries ListObjectsV2 anonymously. 200 = listable; 403 with a readable
|
||||
// known object = object readable only; 404 = no such bucket; 301/307 = region.
|
||||
func (p *Prober) checkS3(ctx context.Context, c model.Candidate) Result {
|
||||
status, header, err := p.do(ctx, "https://"+c.Hostname+"/?list-type=2&max-keys=1")
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
return wrap(classifyError(err))
|
||||
}
|
||||
if status == http.StatusMovedPermanently || status == http.StatusTemporaryRedirect || status == http.StatusPermanentRedirect {
|
||||
if region := header.Get("x-amz-bucket-region"); region != "" {
|
||||
return model.ExposureReachable, "s3 redirect region=" + region
|
||||
return res(model.ExposureReachable, "s3 redirect region="+region)
|
||||
}
|
||||
return model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status)
|
||||
return res(model.ExposureReachable, fmt.Sprintf("s3 redirect status=%d", status))
|
||||
}
|
||||
return classifyStatus(status, "s3")
|
||||
if status >= 200 && status < 300 {
|
||||
return listRead(fmt.Sprintf("s3 status=%d anonymous listing allowed", status))
|
||||
}
|
||||
if status == http.StatusForbidden || status == http.StatusUnauthorized {
|
||||
for _, key := range []string{"index.html", "robots.txt"} {
|
||||
if s, _, err := p.do(ctx, "https://"+c.Hostname+"/"+key); err == nil && s >= 200 && s < 300 {
|
||||
return objRead("s3 object readable (" + key + ") although listing is denied")
|
||||
}
|
||||
}
|
||||
}
|
||||
return wrap(classifyStatus(status, "s3"))
|
||||
}
|
||||
|
||||
// checkGCS uses the public JSON API to test object listing, which requires the
|
||||
// bucket to grant allUsers read access.
|
||||
func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
// checkGCS uses the public JSON API object list (requires storage.objects.list).
|
||||
func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) Result {
|
||||
bucket := c.Resource
|
||||
if bucket == "" {
|
||||
bucket = c.Hostname
|
||||
@@ -316,40 +368,52 @@ func (p *Prober) checkGCS(ctx context.Context, c model.Candidate) (model.Exposur
|
||||
api := "https://storage.googleapis.com/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1"
|
||||
status, _, err := p.do(ctx, api)
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
return wrap(classifyError(err))
|
||||
}
|
||||
return classifyStatus(status, "gcs")
|
||||
if status >= 200 && status < 300 {
|
||||
return listRead(fmt.Sprintf("gcs status=%d anonymous listing allowed", status))
|
||||
}
|
||||
if status == http.StatusForbidden || status == http.StatusUnauthorized {
|
||||
for _, key := range []string{"index.html", "robots.txt"} {
|
||||
obj := "https://storage.googleapis.com/" + url.PathEscape(bucket) + "/" + key
|
||||
if s, _, err := p.do(ctx, obj); err == nil && s >= 200 && s < 300 {
|
||||
return objRead("gcs object readable (" + key + ") although listing is denied")
|
||||
}
|
||||
}
|
||||
}
|
||||
return wrap(classifyStatus(status, "gcs"))
|
||||
}
|
||||
|
||||
// checkFirebase queries the Realtime Database REST endpoint with shallow=true so
|
||||
// only keys are returned. 200 means the security rules allow anonymous reads.
|
||||
func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
// checkFirebase queries the Realtime Database REST endpoint with shallow=true.
|
||||
func (p *Prober) checkFirebase(ctx context.Context, c model.Candidate) Result {
|
||||
status, _, err := p.do(ctx, "https://"+c.Hostname+"/.json?shallow=true")
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
return wrap(classifyError(err))
|
||||
}
|
||||
return classifyStatus(status, "firebase")
|
||||
if status >= 200 && status < 300 {
|
||||
return listRead("firebase rules allow anonymous read")
|
||||
}
|
||||
return wrap(classifyStatus(status, "firebase"))
|
||||
}
|
||||
|
||||
func (p *Prober) checkHTTP(ctx context.Context, endpoint string) (model.ExposureState, string) {
|
||||
func (p *Prober) checkHTTP(ctx context.Context, endpoint string) Result {
|
||||
status, _, err := p.do(ctx, endpoint)
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
return wrap(classifyError(err))
|
||||
}
|
||||
return classifyStatus(status, "")
|
||||
return wrap(classifyStatus(status, ""))
|
||||
}
|
||||
|
||||
// checkTCP reports whether a database endpoint is publicly resolvable and its
|
||||
// port is open. It intentionally does not attempt any protocol handshake or
|
||||
// authentication, so it can never be reported as readable.
|
||||
func (p *Prober) checkTCP(ctx context.Context, host string, port int) (model.ExposureState, string) {
|
||||
// port is open. It never authenticates, so it can never be readable.
|
||||
func (p *Prober) checkTCP(ctx context.Context, host string, port int) Result {
|
||||
d := net.Dialer{Timeout: p.cfg.Timeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, fmt.Sprint(port)))
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
return wrap(classifyError(err))
|
||||
}
|
||||
_ = conn.Close()
|
||||
return model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port)
|
||||
return res(model.ExposureReachable, fmt.Sprintf("tcp/%d open; no authentication attempted", port))
|
||||
}
|
||||
|
||||
// do issues a single GET and reads at most a tiny prefix of the body before
|
||||
@@ -368,6 +432,10 @@ func (p *Prober) do(ctx context.Context, endpoint string) (int, http.Header, err
|
||||
return resp.StatusCode, resp.Header, nil
|
||||
}
|
||||
|
||||
func wrap(state model.ExposureState, detail string) Result {
|
||||
return Result{Exposure: state, Detail: detail}
|
||||
}
|
||||
|
||||
func classifyStatus(status int, tag string) (model.ExposureState, string) {
|
||||
prefix := ""
|
||||
if tag != "" {
|
||||
|
||||
@@ -80,6 +80,24 @@ func TestProbeFirebase(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeAzureContainerDiscovery(t *testing.T) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/images" && r.URL.Query().Get("comp") == "list" {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, h)
|
||||
state, detail := p.Probe(context.Background(), model.Candidate{
|
||||
Hostname: host, Resource: "acct", Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
|
||||
})
|
||||
if state != model.ExposurePublicRead || !strings.Contains(detail, "images") {
|
||||
t.Fatalf("got %s (%s), want public_read via container 'images'", state, detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeDatabaseTCP(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
@@ -97,17 +115,16 @@ func TestProbeDatabaseTCP(t *testing.T) {
|
||||
}()
|
||||
host, port, _ := net.SplitHostPort(ln.Addr().String())
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, detail := p.checkTCP(context.Background(), host, mustAtoi(port))
|
||||
if state != model.ExposureReachable {
|
||||
t.Fatalf("got %s (%s), want reachable", state, detail)
|
||||
r := p.checkTCP(context.Background(), host, mustAtoi(port))
|
||||
if r.Exposure != model.ExposureReachable {
|
||||
t.Fatalf("got %+v, want reachable", r)
|
||||
}
|
||||
if !strings.Contains(detail, "no authentication attempted") {
|
||||
t.Fatalf("detail should be explicit about no auth: %q", detail)
|
||||
if r.ObjectReadable || r.Listable {
|
||||
t.Fatalf("tcp check must not claim readability: %+v", r)
|
||||
}
|
||||
|
||||
// A closed port yields unknown, never readable.
|
||||
state, _ = p.checkTCP(context.Background(), "127.0.0.1", 1)
|
||||
if state == model.ExposurePublicRead {
|
||||
if r := p.checkTCP(context.Background(), "127.0.0.1", 1); r.Exposure == model.ExposurePublicRead {
|
||||
t.Fatal("closed port must never be public_read")
|
||||
}
|
||||
}
|
||||
|
||||
+24
-32
@@ -18,7 +18,7 @@ import (
|
||||
// database endpoint actually answers a read without credentials. No password,
|
||||
// token or credential of any kind is ever sent, and no state is mutated.
|
||||
|
||||
func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureState, string, bool) {
|
||||
func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (Result, bool) {
|
||||
switch c.Service {
|
||||
case model.ServiceRedis, model.ServiceElastiCache, model.ServiceMemoryDB:
|
||||
return p.probeRedis(ctx, c.Hostname, dbPort(c.Service))
|
||||
@@ -31,7 +31,7 @@ func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureS
|
||||
case model.ServiceNeptune:
|
||||
return p.probeHTTPReadable(ctx, "https://"+c.Hostname+"/status")
|
||||
}
|
||||
return model.ExposureUnknown, "", false
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
func dbPort(s model.Service) int {
|
||||
@@ -75,7 +75,7 @@ func (p *Prober) dialTLS(ctx context.Context, host string, port int) (net.Conn,
|
||||
}
|
||||
|
||||
// probeRedis sends PING. Redis answers +PONG when no authentication is required.
|
||||
func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
|
||||
func (p *Prober) probeRedis(ctx context.Context, host string, port int) (Result, bool) {
|
||||
var lastErr error
|
||||
for _, tlsOn := range []bool{false, true} {
|
||||
var (
|
||||
@@ -99,9 +99,9 @@ func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.E
|
||||
resp := string(buf[:n])
|
||||
switch {
|
||||
case strings.HasPrefix(resp, "+PONG"):
|
||||
return model.ExposurePublicRead, "redis PING answered without auth", true
|
||||
return listRead("redis PING answered without auth"), true
|
||||
case strings.Contains(resp, "NOAUTH"), strings.HasPrefix(resp, "-ERR"), strings.Contains(resp, "DENIED"), strings.Contains(resp, "protected mode"):
|
||||
return model.ExposureAuthRequired, "redis requires authentication", true
|
||||
return wrap(model.ExposureAuthRequired, "redis requires authentication"), true
|
||||
}
|
||||
// no/invalid response: try the other transport
|
||||
}
|
||||
@@ -109,17 +109,16 @@ func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.E
|
||||
lastErr = fmt.Errorf("redis: no response")
|
||||
}
|
||||
st, detail := classifyError(lastErr)
|
||||
return st, detail, true
|
||||
return wrap(st, detail), true
|
||||
}
|
||||
|
||||
// probePostgres negotiates TLS if offered, then sends a StartupMessage with no
|
||||
// password. The server's AuthenticationRequest tells us whether it trusts the
|
||||
// connection (AuthenticationOk) or requires a password.
|
||||
func (p *Prober) probePostgres(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
|
||||
func (p *Prober) probePostgres(ctx context.Context, host string, port int) (Result, bool) {
|
||||
conn, err := p.dial(ctx, host, port)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(p.cfg.Timeout))
|
||||
@@ -148,14 +147,12 @@ func (p *Prober) probePostgres(ctx context.Context, host string, port int) (mode
|
||||
binary.BigEndian.PutUint32(msg, uint32(4+len(payload)))
|
||||
copy(msg[4:], payload)
|
||||
if _, err := conn.Write(msg); err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
|
||||
head := make([]byte, 5)
|
||||
if _, err := io.ReadFull(conn, head); err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
kind := head[0]
|
||||
length := int(binary.BigEndian.Uint32(head[1:5]))
|
||||
@@ -165,20 +162,20 @@ func (p *Prober) probePostgres(ctx context.Context, host string, port int) (mode
|
||||
if kind == 'R' && len(body) >= 4 {
|
||||
authType := binary.BigEndian.Uint32(body[:4])
|
||||
if authType == 0 {
|
||||
return model.ExposurePublicRead, "postgres AuthenticationOk (trust, no password required)", true
|
||||
return listRead("postgres AuthenticationOk (trust, no password required)"), true
|
||||
}
|
||||
return model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType), true
|
||||
return wrap(model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType)), true
|
||||
}
|
||||
if kind == 'E' {
|
||||
return model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)", true
|
||||
return wrap(model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)"), true
|
||||
}
|
||||
}
|
||||
return model.ExposureReachable, "postgres port open; auth state unclear", true
|
||||
return wrap(model.ExposureReachable, "postgres port open; auth state unclear"), true
|
||||
}
|
||||
|
||||
// probeMongo performs the OP_MSG handshake then a read-only listDatabases. A
|
||||
// successful, non-empty reply means the deployment has no authentication.
|
||||
func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
|
||||
func (p *Prober) probeMongo(ctx context.Context, host string, port int) (Result, bool) {
|
||||
var (
|
||||
conn net.Conn
|
||||
err error
|
||||
@@ -188,8 +185,7 @@ func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.E
|
||||
if err != nil {
|
||||
conn, err = p.dial(ctx, host, port)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
}
|
||||
defer conn.Close()
|
||||
@@ -197,33 +193,29 @@ func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.E
|
||||
|
||||
doc := bsonDoc{"listDatabases": int32(1), "$db": "admin"}
|
||||
if err := writeOPMsg(conn, doc); err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
raw, err := readOPMsg(conn)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
ok, errmsg := parseBSONFields(raw)
|
||||
switch {
|
||||
case ok == 1 && errmsg == "":
|
||||
return model.ExposurePublicRead, "mongodb listDatabases succeeded without auth", true
|
||||
return listRead("mongodb listDatabases succeeded without auth"), true
|
||||
case strings.Contains(strings.ToLower(errmsg), "auth") || strings.Contains(strings.ToLower(errmsg), "unauthorized"):
|
||||
return model.ExposureAuthRequired, "mongodb requires authentication", true
|
||||
return wrap(model.ExposureAuthRequired, "mongodb requires authentication"), true
|
||||
default:
|
||||
return model.ExposureReachable, "mongodb replied but auth state unclear", true
|
||||
return wrap(model.ExposureReachable, "mongodb replied but auth state unclear"), true
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (model.ExposureState, string, bool) {
|
||||
func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (Result, bool) {
|
||||
status, _, err := p.do(ctx, endpoint)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
return wrap(classifyError(err)), true
|
||||
}
|
||||
st, d := classifyStatus(status, "")
|
||||
return st, d, true
|
||||
return wrap(classifyStatus(status, "")), true
|
||||
}
|
||||
|
||||
// --- minimal BSON/OP_MSG helpers (read-only) ---
|
||||
|
||||
@@ -46,9 +46,9 @@ func TestProbeRedisReadable(t *testing.T) {
|
||||
_, _ = c.Write([]byte("+PONG\r\n"))
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, detail, ok := p.probeRedis(context.Background(), host, port)
|
||||
if !ok || state != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s (%s) ok=%v, want public_read", state, detail, ok)
|
||||
r, ok := p.probeRedis(context.Background(), host, port)
|
||||
if !ok || r.Exposure != model.ExposurePublicRead || !r.ObjectReadable {
|
||||
t.Fatalf("got %+v ok=%v, want public_read/object_readable", r, ok)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,9 +59,9 @@ func TestProbeRedisAuthRequired(t *testing.T) {
|
||||
_, _ = c.Write([]byte("-NOAUTH Authentication required.\r\n"))
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, _, _ := p.probeRedis(context.Background(), host, port)
|
||||
if state != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", state)
|
||||
r, _ := p.probeRedis(context.Background(), host, port)
|
||||
if r.Exposure != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %+v, want auth_required", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,9 +82,9 @@ func TestProbePostgresTrust(t *testing.T) {
|
||||
_, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 0})
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, detail, _ := p.probePostgres(context.Background(), host, port)
|
||||
if state != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s (%s), want public_read", state, detail)
|
||||
r, _ := p.probePostgres(context.Background(), host, port)
|
||||
if r.Exposure != model.ExposurePublicRead || !r.ObjectReadable {
|
||||
t.Fatalf("got %+v, want public_read/object_readable", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,9 +105,9 @@ func TestProbePostgresPassword(t *testing.T) {
|
||||
_, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 5})
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, _, _ := p.probePostgres(context.Background(), host, port)
|
||||
if state != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", state)
|
||||
r, _ := p.probePostgres(context.Background(), host, port)
|
||||
if r.Exposure != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %+v, want auth_required", r)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user