add read-only web UI and per-vendor readability prober
Web UI (internal/webui): - dependency-free embedded HTML/JS served on the loopback listener - /api/assets (provider/service/exposure/q filters) and /api/stats - optional HTTP basic auth via UI_USER/UI_PASS Prober (internal/probe): - vendor-specific, unauthenticated, non-destructive checks: S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list, Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for database endpoints (never reported readable, no auth attempted) - exposure_detail records the factual reason (status code, region, tcp port) - bounded random-sample queue instead of FIFO: one high-volume hostname template (mrap) no longer starves every other asset type - schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER) Live: stable ~450MiB under 1GiB, no restarts, UI and probe states (private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
1 parent
9710efec1a
commit
8fce55f8c9
13 files changed
+843
-72
No files matched your search
+102
-16
@@ -1,7 +1,9 @@
|
||||
package probe
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -11,20 +13,18 @@ import (
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
func newProber(t *testing.T, status int) (*Prober, string) {
|
||||
func serverHost(t *testing.T, h http.Handler) string {
|
||||
t.Helper()
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
srv := httptest.NewTLSServer(h)
|
||||
t.Cleanup(srv.Close)
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
// httptest uses a self-signed cert; the prober already skips verification.
|
||||
_ = tls.Config{}
|
||||
host := strings.TrimPrefix(srv.URL, "https://")
|
||||
return p, host
|
||||
return strings.TrimPrefix(srv.URL, "https://")
|
||||
}
|
||||
|
||||
func TestProbeClassification(t *testing.T) {
|
||||
func statusHandler(status int) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) })
|
||||
}
|
||||
|
||||
func TestProbeStorageStatuses(t *testing.T) {
|
||||
cases := []struct {
|
||||
status int
|
||||
want model.ExposureState
|
||||
@@ -35,11 +35,97 @@ func TestProbeClassification(t *testing.T) {
|
||||
{http.StatusNotFound, model.ExposureNotFound},
|
||||
{http.StatusInternalServerError, model.ExposureReachable},
|
||||
}
|
||||
for _, c := range cases {
|
||||
p, host := newProber(t, c.status)
|
||||
got := p.probe(t.Context(), model.Candidate{Hostname: host})
|
||||
if got != c.want {
|
||||
t.Errorf("status %d: got %s, want %s", c.status, got, c.want)
|
||||
for _, tc := range cases {
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, statusHandler(tc.status))
|
||||
got, detail := p.Probe(context.Background(), model.Candidate{
|
||||
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
||||
})
|
||||
if got != tc.want {
|
||||
t.Errorf("status %d: got %s (%s), want %s", tc.status, got, detail, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeS3RegionRedirect(t *testing.T) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("x-amz-bucket-region", "eu-central-1")
|
||||
w.WriteHeader(http.StatusMovedPermanently)
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, h)
|
||||
got, detail := p.Probe(context.Background(), model.Candidate{
|
||||
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
||||
})
|
||||
if got != model.ExposureReachable || !strings.Contains(detail, "eu-central-1") {
|
||||
t.Fatalf("got %s (%s), want reachable with region", got, detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeFirebase(t *testing.T) {
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, statusHandler(http.StatusUnauthorized))
|
||||
got, _ := p.Probe(context.Background(), model.Candidate{
|
||||
Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
|
||||
})
|
||||
if got != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", got)
|
||||
}
|
||||
host2 := serverHost(t, statusHandler(http.StatusOK))
|
||||
got, _ = p.Probe(context.Background(), model.Candidate{
|
||||
Hostname: host2, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
|
||||
})
|
||||
if got != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s, want public_read", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeDatabaseTCP(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
host, port, _ := net.SplitHostPort(ln.Addr().String())
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, detail := p.checkTCP(context.Background(), host, mustAtoi(port))
|
||||
if state != model.ExposureReachable {
|
||||
t.Fatalf("got %s (%s), want reachable", state, detail)
|
||||
}
|
||||
if !strings.Contains(detail, "no authentication attempted") {
|
||||
t.Fatalf("detail should be explicit about no auth: %q", detail)
|
||||
}
|
||||
|
||||
// A closed port yields unknown, never readable.
|
||||
state, _ = p.checkTCP(context.Background(), "127.0.0.1", 1)
|
||||
if state == model.ExposurePublicRead {
|
||||
t.Fatal("closed port must never be public_read")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyErrorNXDOMAIN(t *testing.T) {
|
||||
state, detail := classifyError(&net.DNSError{Err: "no such host", IsNotFound: true})
|
||||
if state != model.ExposurePrivate || !strings.Contains(detail, "no public record") {
|
||||
t.Fatalf("got %s (%s)", state, detail)
|
||||
}
|
||||
if state, _ := classifyError(errors.New("boom")); state != model.ExposureUnknown {
|
||||
t.Fatalf("got %s, want unknown", state)
|
||||
}
|
||||
}
|
||||
|
||||
func mustAtoi(s string) int {
|
||||
n := 0
|
||||
for _, c := range s {
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
Reference in new issue
Block a user