add read-only web UI and per-vendor readability prober
Web UI (internal/webui): - dependency-free embedded HTML/JS served on the loopback listener - /api/assets (provider/service/exposure/q filters) and /api/stats - optional HTTP basic auth via UI_USER/UI_PASS Prober (internal/probe): - vendor-specific, unauthenticated, non-destructive checks: S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list, Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for database endpoints (never reported readable, no auth attempted) - exposure_detail records the factual reason (status code, region, tcp port) - bounded random-sample queue instead of FIFO: one high-volume hostname template (mrap) no longer starves every other asset type - schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER) Live: stable ~450MiB under 1GiB, no restarts, UI and probe states (private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
1 parent
9710efec1a
commit
8fce55f8c9
13 files changed
+843
-72
No files matched your search
@@ -0,0 +1,94 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
lastFilter store.AssetFilter
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListAssets(ctx context.Context, fl store.AssetFilter) ([]store.Asset, error) {
|
||||
f.lastFilter = fl
|
||||
return []store.Asset{{ID: 1, Provider: "aws", Service: "s3", ProviderHost: "b.s3.amazonaws.com", Exposure: "public_read"}}, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Stats(ctx context.Context) (store.Stats, error) {
|
||||
return store.Stats{Total: 1, Readable: 1}, nil
|
||||
}
|
||||
|
||||
func TestAPI(t *testing.T) {
|
||||
fs := &fakeStore{}
|
||||
srv := httptest.NewServer(New(fs, Config{}))
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/api/assets?provider=aws&q=bucket&limit=5")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
var assets []store.Asset
|
||||
if err := json.NewDecoder(resp.Body).Decode(&assets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(assets) != 1 || fs.lastFilter.Provider != "aws" || fs.lastFilter.Query != "bucket" || fs.lastFilter.Limit != 5 {
|
||||
t.Fatalf("unexpected assets/filter: %+v %+v", assets, fs.lastFilter)
|
||||
}
|
||||
|
||||
statsResp, err := http.Get(srv.URL + "/api/stats")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer statsResp.Body.Close()
|
||||
var st store.Stats
|
||||
if err := json.NewDecoder(statsResp.Body).Decode(&st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.Total != 1 {
|
||||
t.Fatalf("stats = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicAuth(t *testing.T) {
|
||||
srv := httptest.NewServer(New(&fakeStore{}, Config{User: "u", Pass: "p"}))
|
||||
defer srv.Close()
|
||||
|
||||
resp, _ := http.Get(srv.URL + "/api/stats")
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", resp.StatusCode)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
req, _ := http.NewRequest(http.MethodGet, srv.URL+"/api/stats", nil)
|
||||
req.SetBasicAuth("u", "p")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexServed(t *testing.T) {
|
||||
srv := httptest.NewServer(New(&fakeStore{}, Config{}))
|
||||
defer srv.Close()
|
||||
resp, err := http.Get(srv.URL + "/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "text/html; charset=utf-8" {
|
||||
t.Fatalf("content-type = %q", ct)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user