add read-only web UI and per-vendor readability prober

Web UI (internal/webui):
- dependency-free embedded HTML/JS served on the loopback listener
- /api/assets (provider/service/exposure/q filters) and /api/stats
- optional HTTP basic auth via UI_USER/UI_PASS

Prober (internal/probe):
- vendor-specific, unauthenticated, non-destructive checks:
  S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list,
  Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for
  database endpoints (never reported readable, no auth attempted)
- exposure_detail records the factual reason (status code, region, tcp port)
- bounded random-sample queue instead of FIFO: one high-volume hostname
  template (mrap) no longer starves every other asset type
- schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER)

Live: stable ~450MiB under 1GiB, no restarts, UI and probe states
(private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
StasonJatham committed 2026-09-22 13:58:28 +02:00
1 parent 9710efec1a
commit 8fce55f8c9
13 files changed
+843 -72

No files matched your search

+102 -16
View File
@@ -1,7 +1,9 @@
package probe
import (
"crypto/tls"
"context"
"errors"
"net"
"net/http"
"net/http/httptest"
"strings"
@@ -11,20 +13,18 @@ import (
"tea.karlcom.de/karl/bucketeer/internal/model"
)
func newProber(t *testing.T, status int) (*Prober, string) {
func serverHost(t *testing.T, h http.Handler) string {
t.Helper()
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(status)
}))
srv := httptest.NewTLSServer(h)
t.Cleanup(srv.Close)
p := New(Config{Workers: 1, Timeout: time.Second})
// httptest uses a self-signed cert; the prober already skips verification.
_ = tls.Config{}
host := strings.TrimPrefix(srv.URL, "https://")
return p, host
return strings.TrimPrefix(srv.URL, "https://")
}
func TestProbeClassification(t *testing.T) {
func statusHandler(status int) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) })
}
func TestProbeStorageStatuses(t *testing.T) {
cases := []struct {
status int
want model.ExposureState
@@ -35,11 +35,97 @@ func TestProbeClassification(t *testing.T) {
{http.StatusNotFound, model.ExposureNotFound},
{http.StatusInternalServerError, model.ExposureReachable},
}
for _, c := range cases {
p, host := newProber(t, c.status)
got := p.probe(t.Context(), model.Candidate{Hostname: host})
if got != c.want {
t.Errorf("status %d: got %s, want %s", c.status, got, c.want)
for _, tc := range cases {
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, statusHandler(tc.status))
got, detail := p.Probe(context.Background(), model.Candidate{
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
})
if got != tc.want {
t.Errorf("status %d: got %s (%s), want %s", tc.status, got, detail, tc.want)
}
}
}
func TestProbeS3RegionRedirect(t *testing.T) {
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("x-amz-bucket-region", "eu-central-1")
w.WriteHeader(http.StatusMovedPermanently)
})
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, h)
got, detail := p.Probe(context.Background(), model.Candidate{
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
})
if got != model.ExposureReachable || !strings.Contains(detail, "eu-central-1") {
t.Fatalf("got %s (%s), want reachable with region", got, detail)
}
}
func TestProbeFirebase(t *testing.T) {
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, statusHandler(http.StatusUnauthorized))
got, _ := p.Probe(context.Background(), model.Candidate{
Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
})
if got != model.ExposureAuthRequired {
t.Fatalf("got %s, want auth_required", got)
}
host2 := serverHost(t, statusHandler(http.StatusOK))
got, _ = p.Probe(context.Background(), model.Candidate{
Hostname: host2, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
})
if got != model.ExposurePublicRead {
t.Fatalf("got %s, want public_read", got)
}
}
func TestProbeDatabaseTCP(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
conn.Close()
}
}()
host, port, _ := net.SplitHostPort(ln.Addr().String())
p := New(Config{Workers: 1, Timeout: time.Second})
state, detail := p.checkTCP(context.Background(), host, mustAtoi(port))
if state != model.ExposureReachable {
t.Fatalf("got %s (%s), want reachable", state, detail)
}
if !strings.Contains(detail, "no authentication attempted") {
t.Fatalf("detail should be explicit about no auth: %q", detail)
}
// A closed port yields unknown, never readable.
state, _ = p.checkTCP(context.Background(), "127.0.0.1", 1)
if state == model.ExposurePublicRead {
t.Fatal("closed port must never be public_read")
}
}
func TestClassifyErrorNXDOMAIN(t *testing.T) {
state, detail := classifyError(&net.DNSError{Err: "no such host", IsNotFound: true})
if state != model.ExposurePrivate || !strings.Contains(detail, "no public record") {
t.Fatalf("got %s (%s)", state, detail)
}
if state, _ := classifyError(errors.New("boom")); state != model.ExposureUnknown {
t.Fatalf("got %s, want unknown", state)
}
}
func mustAtoi(s string) int {
n := 0
for _, c := range s {
n = n*10 + int(c-'0')
}
return n
}