add read-only web UI and per-vendor readability prober
Web UI (internal/webui): - dependency-free embedded HTML/JS served on the loopback listener - /api/assets (provider/service/exposure/q filters) and /api/stats - optional HTTP basic auth via UI_USER/UI_PASS Prober (internal/probe): - vendor-specific, unauthenticated, non-destructive checks: S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list, Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for database endpoints (never reported readable, no auth attempted) - exposure_detail records the factual reason (status code, region, tcp port) - bounded random-sample queue instead of FIFO: one high-volume hostname template (mrap) no longer starves every other asset type - schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER) Live: stable ~450MiB under 1GiB, no restarts, UI and probe states (private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
1 parent
9710efec1a
commit
8fce55f8c9
13 files changed
+843
-72
No files matched your search
+8
-4
@@ -24,6 +24,7 @@ import (
|
||||
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/probe"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/webui"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -87,9 +88,10 @@ func run() error {
|
||||
prober = probe.New(probe.Config{
|
||||
Workers: cfg.ProbeWorkers,
|
||||
Timeout: cfg.ProbeTimeout,
|
||||
Queue: cfg.ProbeQueueSize,
|
||||
Metrics: m,
|
||||
Update: func(uctx context.Context, c model.Candidate, state model.ExposureState) {
|
||||
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state)
|
||||
Update: func(uctx context.Context, c model.Candidate, state model.ExposureState, detail string) {
|
||||
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state, detail)
|
||||
},
|
||||
})
|
||||
prober.Start(ctx)
|
||||
@@ -137,7 +139,8 @@ func run() error {
|
||||
monitorDone <- monitor.Run(ctx, monitorCfg)
|
||||
}()
|
||||
|
||||
httpSrv := startHTTP(cfg.MetricsAddr, m, st, &monitorReady)
|
||||
ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass})
|
||||
httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady)
|
||||
go logStats(ctx, m, cfg.StatsInterval)
|
||||
|
||||
slog.Info("ctcloud started")
|
||||
@@ -172,9 +175,10 @@ func run() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ready *atomic.Bool) *http.Server {
|
||||
func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ui http.Handler, ready *atomic.Bool) *http.Server {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", m.Handler())
|
||||
mux.Handle("/", ui)
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok\n"))
|
||||
|
||||
Reference in new issue
Block a user