add read-only web UI and per-vendor readability prober

Web UI (internal/webui):
- dependency-free embedded HTML/JS served on the loopback listener
- /api/assets (provider/service/exposure/q filters) and /api/stats
- optional HTTP basic auth via UI_USER/UI_PASS

Prober (internal/probe):
- vendor-specific, unauthenticated, non-destructive checks:
  S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list,
  Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for
  database endpoints (never reported readable, no auth attempted)
- exposure_detail records the factual reason (status code, region, tcp port)
- bounded random-sample queue instead of FIFO: one high-volume hostname
  template (mrap) no longer starves every other asset type
- schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER)

Live: stable ~450MiB under 1GiB, no restarts, UI and probe states
(private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
StasonJatham committed 2026-09-22 13:58:28 +02:00
1 parent 9710efec1a
commit 8fce55f8c9
13 files changed
+843 -72

No files matched your search

+8 -4
View File
@@ -24,6 +24,7 @@ import (
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
"tea.karlcom.de/karl/bucketeer/internal/probe"
"tea.karlcom.de/karl/bucketeer/internal/store"
"tea.karlcom.de/karl/bucketeer/internal/webui"
)
func main() {
@@ -87,9 +88,10 @@ func run() error {
prober = probe.New(probe.Config{
Workers: cfg.ProbeWorkers,
Timeout: cfg.ProbeTimeout,
Queue: cfg.ProbeQueueSize,
Metrics: m,
Update: func(uctx context.Context, c model.Candidate, state model.ExposureState) {
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state)
Update: func(uctx context.Context, c model.Candidate, state model.ExposureState, detail string) {
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state, detail)
},
})
prober.Start(ctx)
@@ -137,7 +139,8 @@ func run() error {
monitorDone <- monitor.Run(ctx, monitorCfg)
}()
httpSrv := startHTTP(cfg.MetricsAddr, m, st, &monitorReady)
ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass})
httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady)
go logStats(ctx, m, cfg.StatsInterval)
slog.Info("ctcloud started")
@@ -172,9 +175,10 @@ func run() error {
return nil
}
func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ready *atomic.Bool) *http.Server {
func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ui http.Handler, ready *atomic.Bool) *http.Server {
mux := http.NewServeMux()
mux.Handle("/metrics", m.Handler())
mux.Handle("/", ui)
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok\n"))