Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
149 lines
4.4 KiB
Go
149 lines
4.4 KiB
Go
package probe
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
)
|
|
|
|
func serverHost(t *testing.T, h http.Handler) string {
|
|
t.Helper()
|
|
srv := httptest.NewTLSServer(h)
|
|
t.Cleanup(srv.Close)
|
|
return strings.TrimPrefix(srv.URL, "https://")
|
|
}
|
|
|
|
func statusHandler(status int) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) })
|
|
}
|
|
|
|
func TestProbeStorageStatuses(t *testing.T) {
|
|
cases := []struct {
|
|
status int
|
|
want model.ExposureState
|
|
}{
|
|
{http.StatusOK, model.ExposurePublicRead},
|
|
{http.StatusForbidden, model.ExposureAuthRequired},
|
|
{http.StatusUnauthorized, model.ExposureAuthRequired},
|
|
{http.StatusNotFound, model.ExposureNotFound},
|
|
{http.StatusInternalServerError, model.ExposureReachable},
|
|
}
|
|
for _, tc := range cases {
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
host := serverHost(t, statusHandler(tc.status))
|
|
got, detail := p.Probe(context.Background(), model.Candidate{
|
|
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
|
})
|
|
if got != tc.want {
|
|
t.Errorf("status %d: got %s (%s), want %s", tc.status, got, detail, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProbeS3RegionRedirect(t *testing.T) {
|
|
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("x-amz-bucket-region", "eu-central-1")
|
|
w.WriteHeader(http.StatusMovedPermanently)
|
|
})
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
host := serverHost(t, h)
|
|
got, detail := p.Probe(context.Background(), model.Candidate{
|
|
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
|
})
|
|
if got != model.ExposureReachable || !strings.Contains(detail, "eu-central-1") {
|
|
t.Fatalf("got %s (%s), want reachable with region", got, detail)
|
|
}
|
|
}
|
|
|
|
func TestProbeFirebase(t *testing.T) {
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
host := serverHost(t, statusHandler(http.StatusUnauthorized))
|
|
got, _ := p.Probe(context.Background(), model.Candidate{
|
|
Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
|
|
})
|
|
if got != model.ExposureAuthRequired {
|
|
t.Fatalf("got %s, want auth_required", got)
|
|
}
|
|
host2 := serverHost(t, statusHandler(http.StatusOK))
|
|
got, _ = p.Probe(context.Background(), model.Candidate{
|
|
Hostname: host2, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
|
|
})
|
|
if got != model.ExposurePublicRead {
|
|
t.Fatalf("got %s, want public_read", got)
|
|
}
|
|
}
|
|
|
|
func TestProbeAzureContainerDiscovery(t *testing.T) {
|
|
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/images" && r.URL.Query().Get("comp") == "list" {
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNotFound)
|
|
})
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
host := serverHost(t, h)
|
|
state, detail := p.Probe(context.Background(), model.Candidate{
|
|
Hostname: host, Resource: "acct", Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
|
|
})
|
|
if state != model.ExposurePublicRead || !strings.Contains(detail, "images") {
|
|
t.Fatalf("got %s (%s), want public_read via container 'images'", state, detail)
|
|
}
|
|
}
|
|
|
|
func TestProbeDatabaseTCP(t *testing.T) {
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer ln.Close()
|
|
go func() {
|
|
for {
|
|
conn, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
conn.Close()
|
|
}
|
|
}()
|
|
host, port, _ := net.SplitHostPort(ln.Addr().String())
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
r := p.checkTCP(context.Background(), host, mustAtoi(port))
|
|
if r.Exposure != model.ExposureReachable {
|
|
t.Fatalf("got %+v, want reachable", r)
|
|
}
|
|
if r.ObjectReadable || r.Listable {
|
|
t.Fatalf("tcp check must not claim readability: %+v", r)
|
|
}
|
|
|
|
// A closed port yields unknown, never readable.
|
|
if r := p.checkTCP(context.Background(), "127.0.0.1", 1); r.Exposure == model.ExposurePublicRead {
|
|
t.Fatal("closed port must never be public_read")
|
|
}
|
|
}
|
|
|
|
func TestClassifyErrorNXDOMAIN(t *testing.T) {
|
|
state, detail := classifyError(&net.DNSError{Err: "no such host", IsNotFound: true})
|
|
if state != model.ExposurePrivate || !strings.Contains(detail, "no public record") {
|
|
t.Fatalf("got %s (%s)", state, detail)
|
|
}
|
|
if state, _ := classifyError(errors.New("boom")); state != model.ExposureUnknown {
|
|
t.Fatalf("got %s, want unknown", state)
|
|
}
|
|
}
|
|
|
|
func mustAtoi(s string) int {
|
|
n := 0
|
|
for _, c := range s {
|
|
n = n*10 + int(c-'0')
|
|
}
|
|
return n
|
|
}
|