add read-only web UI and per-vendor readability prober

Web UI (internal/webui):
- dependency-free embedded HTML/JS served on the loopback listener
- /api/assets (provider/service/exposure/q filters) and /api/stats
- optional HTTP basic auth via UI_USER/UI_PASS

Prober (internal/probe):
- vendor-specific, unauthenticated, non-destructive checks:
  S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list,
  Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for
  database endpoints (never reported readable, no auth attempted)
- exposure_detail records the factual reason (status code, region, tcp port)
- bounded random-sample queue instead of FIFO: one high-volume hostname
  template (mrap) no longer starves every other asset type
- schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER)

Live: stable ~450MiB under 1GiB, no restarts, UI and probe states
(private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
StasonJatham committed 2026-09-22 13:58:28 +02:00
1 parent 9710efec1a
commit 8fce55f8c9
13 files changed
+843 -72

No files matched your search

+115 -4
View File
@@ -138,9 +138,120 @@ func nonNil(s []string) []string {
}
// UpdateExposure records the outcome of an optional reachability probe.
func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState) error {
_, err := s.pool.Exec(ctx,
`UPDATE cloud_assets SET exposure_state = $4 WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
provider, service, providerHostname, state.String())
func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState, detail string) error {
_, err := s.pool.Exec(ctx, `
UPDATE cloud_assets
SET exposure_state = $4, exposure_detail = $5, probed_at = now()
WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
provider, service, providerHostname, state.String(), detail)
return err
}
// Asset is a row of the cloud_assets table for the read-only UI.
type Asset struct {
ID int64 `json:"id"`
Provider string `json:"provider"`
Service string `json:"service"`
Hostname string `json:"hostname"`
ProviderHost string `json:"provider_hostname"`
Resource string `json:"resource_identifier"`
Region string `json:"region"`
PublicSemantic string `json:"public_semantic"`
Exposure string `json:"exposure_state"`
ExposureDetail string `json:"exposure_detail"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
ProbedAt *time.Time `json:"probed_at"`
}
// AssetFilter selects assets for the UI. Empty fields are ignored.
type AssetFilter struct {
Provider string
Service string
Exposure string
Query string
Limit int
Offset int
}
// ListAssets returns assets matching the filter, newest first.
func (s *Store) ListAssets(ctx context.Context, f AssetFilter) ([]Asset, error) {
if f.Limit <= 0 || f.Limit > 500 {
f.Limit = 100
}
rows, err := s.pool.Query(ctx, `
SELECT id, provider, service, hostname, provider_hostname, resource_identifier,
region, public_semantic, exposure_state, exposure_detail, first_seen, last_seen, probed_at
FROM cloud_assets
WHERE ($1 = '' OR provider = $1)
AND ($2 = '' OR service = $2)
AND ($3 = '' OR exposure_state = $3)
AND ($4 = '' OR provider_hostname ILIKE '%' || $4 || '%' OR hostname ILIKE '%' || $4 || '%' OR resource_identifier ILIKE '%' || $4 || '%')
ORDER BY last_seen DESC
LIMIT $5 OFFSET $6`,
f.Provider, f.Service, f.Exposure, f.Query, f.Limit, f.Offset)
if err != nil {
return nil, err
}
defer rows.Close()
out := []Asset{}
for rows.Next() {
var a Asset
if err := rows.Scan(&a.ID, &a.Provider, &a.Service, &a.Hostname, &a.ProviderHost,
&a.Resource, &a.Region, &a.PublicSemantic, &a.Exposure, &a.ExposureDetail,
&a.FirstSeen, &a.LastSeen, &a.ProbedAt); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
// Count is a (label, count) pair for the UI summary.
type Count struct {
Label string `json:"label"`
Count int64 `json:"count"`
}
// Stats summarizes the assets table.
type Stats struct {
Total int64 `json:"total"`
Readable int64 `json:"readable"`
ByProvider []Count `json:"by_provider"`
ByService []Count `json:"by_service"`
ByExposure []Count `json:"by_exposure"`
}
func (s *Store) Stats(ctx context.Context) (Stats, error) {
var st Stats
if err := s.pool.QueryRow(ctx, `SELECT count(*), count(*) FILTER (WHERE exposure_state = 'public_read') FROM cloud_assets`).Scan(&st.Total, &st.Readable); err != nil {
return st, err
}
group := func(column string) ([]Count, error) {
rows, err := s.pool.Query(ctx, `SELECT `+column+`, count(*) FROM cloud_assets GROUP BY 1 ORDER BY 2 DESC`)
if err != nil {
return nil, err
}
defer rows.Close()
var counts []Count
for rows.Next() {
var c Count
if err := rows.Scan(&c.Label, &c.Count); err != nil {
return nil, err
}
counts = append(counts, c)
}
return counts, rows.Err()
}
var err error
if st.ByProvider, err = group("provider"); err != nil {
return st, err
}
if st.ByService, err = group("service"); err != nil {
return st, err
}
if st.ByExposure, err = group("exposure_state"); err != nil {
return st, err
}
return st, nil
}