add read-only web UI and per-vendor readability prober
Web UI (internal/webui): - dependency-free embedded HTML/JS served on the loopback listener - /api/assets (provider/service/exposure/q filters) and /api/stats - optional HTTP basic auth via UI_USER/UI_PASS Prober (internal/probe): - vendor-specific, unauthenticated, non-destructive checks: S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list, Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for database endpoints (never reported readable, no auth attempted) - exposure_detail records the factual reason (status code, region, tcp port) - bounded random-sample queue instead of FIFO: one high-volume hostname template (mrap) no longer starves every other asset type - schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER) Live: stable ~450MiB under 1GiB, no restarts, UI and probe states (private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
1 parent
9710efec1a
commit
8fce55f8c9
13 files changed
+843
-72
No files matched your search
+115
-4
@@ -138,9 +138,120 @@ func nonNil(s []string) []string {
|
||||
}
|
||||
|
||||
// UpdateExposure records the outcome of an optional reachability probe.
|
||||
func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE cloud_assets SET exposure_state = $4 WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
|
||||
provider, service, providerHostname, state.String())
|
||||
func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState, detail string) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE cloud_assets
|
||||
SET exposure_state = $4, exposure_detail = $5, probed_at = now()
|
||||
WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
|
||||
provider, service, providerHostname, state.String(), detail)
|
||||
return err
|
||||
}
|
||||
|
||||
// Asset is a row of the cloud_assets table for the read-only UI.
|
||||
type Asset struct {
|
||||
ID int64 `json:"id"`
|
||||
Provider string `json:"provider"`
|
||||
Service string `json:"service"`
|
||||
Hostname string `json:"hostname"`
|
||||
ProviderHost string `json:"provider_hostname"`
|
||||
Resource string `json:"resource_identifier"`
|
||||
Region string `json:"region"`
|
||||
PublicSemantic string `json:"public_semantic"`
|
||||
Exposure string `json:"exposure_state"`
|
||||
ExposureDetail string `json:"exposure_detail"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
ProbedAt *time.Time `json:"probed_at"`
|
||||
}
|
||||
|
||||
// AssetFilter selects assets for the UI. Empty fields are ignored.
|
||||
type AssetFilter struct {
|
||||
Provider string
|
||||
Service string
|
||||
Exposure string
|
||||
Query string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
// ListAssets returns assets matching the filter, newest first.
|
||||
func (s *Store) ListAssets(ctx context.Context, f AssetFilter) ([]Asset, error) {
|
||||
if f.Limit <= 0 || f.Limit > 500 {
|
||||
f.Limit = 100
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, provider, service, hostname, provider_hostname, resource_identifier,
|
||||
region, public_semantic, exposure_state, exposure_detail, first_seen, last_seen, probed_at
|
||||
FROM cloud_assets
|
||||
WHERE ($1 = '' OR provider = $1)
|
||||
AND ($2 = '' OR service = $2)
|
||||
AND ($3 = '' OR exposure_state = $3)
|
||||
AND ($4 = '' OR provider_hostname ILIKE '%' || $4 || '%' OR hostname ILIKE '%' || $4 || '%' OR resource_identifier ILIKE '%' || $4 || '%')
|
||||
ORDER BY last_seen DESC
|
||||
LIMIT $5 OFFSET $6`,
|
||||
f.Provider, f.Service, f.Exposure, f.Query, f.Limit, f.Offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []Asset{}
|
||||
for rows.Next() {
|
||||
var a Asset
|
||||
if err := rows.Scan(&a.ID, &a.Provider, &a.Service, &a.Hostname, &a.ProviderHost,
|
||||
&a.Resource, &a.Region, &a.PublicSemantic, &a.Exposure, &a.ExposureDetail,
|
||||
&a.FirstSeen, &a.LastSeen, &a.ProbedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Count is a (label, count) pair for the UI summary.
|
||||
type Count struct {
|
||||
Label string `json:"label"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
// Stats summarizes the assets table.
|
||||
type Stats struct {
|
||||
Total int64 `json:"total"`
|
||||
Readable int64 `json:"readable"`
|
||||
ByProvider []Count `json:"by_provider"`
|
||||
ByService []Count `json:"by_service"`
|
||||
ByExposure []Count `json:"by_exposure"`
|
||||
}
|
||||
|
||||
func (s *Store) Stats(ctx context.Context) (Stats, error) {
|
||||
var st Stats
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*), count(*) FILTER (WHERE exposure_state = 'public_read') FROM cloud_assets`).Scan(&st.Total, &st.Readable); err != nil {
|
||||
return st, err
|
||||
}
|
||||
group := func(column string) ([]Count, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT `+column+`, count(*) FROM cloud_assets GROUP BY 1 ORDER BY 2 DESC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var counts []Count
|
||||
for rows.Next() {
|
||||
var c Count
|
||||
if err := rows.Scan(&c.Label, &c.Count); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
counts = append(counts, c)
|
||||
}
|
||||
return counts, rows.Err()
|
||||
}
|
||||
var err error
|
||||
if st.ByProvider, err = group("provider"); err != nil {
|
||||
return st, err
|
||||
}
|
||||
if st.ByService, err = group("service"); err != nil {
|
||||
return st, err
|
||||
}
|
||||
if st.ByExposure, err = group("exposure_state"); err != nil {
|
||||
return st, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
Reference in new issue
Block a user