Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs
Logging: - structured slog with LOG_FORMAT=json|text and a stable key vocabulary - periodic stats record with queue depths, db pool, goroutines, memory, dns/probe - no per-certificate logging (hot path is counters only) UI (Bucketeer branding, DESIGN.md): - light+dark themes (OS default, persisted header toggle, no first-paint flash) - official mascot: lossless WebP light/dark logos + favicon, embedded and served with cache headers (1.2MB source PNGs -> ~27/22KB webp) - Assets/Contents views, pagination (offset) and native virtual scrolling (content-visibility) so large result sets do not blow up the DOM - search with debounce/abort + URL state, exposure chips, sort, summary counts Readability + indexing: - internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch (metadata only: keys/sizes/dates; never object contents) - asset_entries table + store write/search; readable resources are indexed and searchable in the UI - deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING, PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune HTTP reads - Azure account-level listing fixed: 403 is reported reachable, not auth_required, since anonymous access is per-container DNS: - curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at once broke the container resolver) - random healthy selection + failover + per-server cooldown Other: prober Close fixes a shutdown hang; config/README updated.
This commit is contained in:
StasonJatham committed
1 parent
8fce55f8c9
commit
a366905a43
28 files changed
+9744
-241
No files matched your search
+157
-2
@@ -49,6 +49,12 @@ func (s *Store) Close() { s.pool.Close() }
|
||||
|
||||
func (s *Store) Pool() *pgxpool.Pool { return s.pool }
|
||||
|
||||
// PoolStats reports pgx pool utilization for logging/metrics.
|
||||
func (s *Store) PoolStats() (total, idle, acquired int32) {
|
||||
st := s.pool.Stat()
|
||||
return st.TotalConns(), st.IdleConns(), st.AcquiredConns()
|
||||
}
|
||||
|
||||
// Migrate applies the embedded schema idempotently.
|
||||
func (s *Store) Migrate(ctx context.Context) error {
|
||||
for _, name := range migrations.SQL() {
|
||||
@@ -137,6 +143,129 @@ func nonNil(s []string) []string {
|
||||
return s
|
||||
}
|
||||
|
||||
// IndexEntry is one indexed resource item for the UI.
|
||||
type IndexEntry struct {
|
||||
ID int64 `json:"id"`
|
||||
AssetID int64 `json:"asset_id"`
|
||||
Provider string `json:"provider"`
|
||||
Service string `json:"service"`
|
||||
ProviderHost string `json:"provider_hostname"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Updated *time.Time `json:"updated"`
|
||||
ContentType string `json:"content_type"`
|
||||
IndexedAt time.Time `json:"indexed_at"`
|
||||
}
|
||||
|
||||
// ReplaceEntries replaces the stored index for a readable asset. Returns the
|
||||
// number of entries stored; a no-op if the asset row is unknown or entries is
|
||||
// empty.
|
||||
func (s *Store) ReplaceEntries(ctx context.Context, provider, service, host string, entries []model.Entry) (int, error) {
|
||||
if len(entries) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var assetID int64
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT id FROM cloud_assets WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
|
||||
provider, service, host).Scan(&assetID)
|
||||
if err == pgx.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM asset_entries WHERE asset_id = $1`, assetID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
b := &pgx.Batch{}
|
||||
for i := range entries {
|
||||
e := &entries[i]
|
||||
var updated any
|
||||
if !e.Updated.IsZero() {
|
||||
updated = e.Updated
|
||||
}
|
||||
b.Queue(`
|
||||
INSERT INTO asset_entries (asset_id, kind, name, size, updated, content_type, etag)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (asset_id, kind, name) DO UPDATE
|
||||
SET size = EXCLUDED.size, updated = EXCLUDED.updated,
|
||||
content_type = EXCLUDED.content_type, etag = EXCLUDED.etag, indexed_at = now()`,
|
||||
assetID, e.Kind, e.Name, e.Size, updated, e.ContentType, e.ETag)
|
||||
}
|
||||
br := tx.SendBatch(ctx, b)
|
||||
for range entries {
|
||||
if _, err := br.Exec(); err != nil {
|
||||
br.Close()
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := br.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(entries), nil
|
||||
}
|
||||
|
||||
// SearchEntries searches the index across all readable assets, newest first.
|
||||
func (s *Store) SearchEntries(ctx context.Context, query string, limit, offset int) ([]IndexEntry, int, error) {
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.id, e.asset_id, a.provider, a.service, a.provider_hostname,
|
||||
e.kind, e.name, e.size, e.updated, e.content_type, e.indexed_at
|
||||
FROM asset_entries e JOIN cloud_assets a ON a.id = e.asset_id
|
||||
WHERE ($1 = '' OR e.name ILIKE '%' || $1 || '%' OR a.provider_hostname ILIKE '%' || $1 || '%')
|
||||
ORDER BY e.indexed_at DESC, e.id DESC
|
||||
LIMIT $2 OFFSET $3`, query, limit, offset)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []IndexEntry{}
|
||||
for rows.Next() {
|
||||
var e IndexEntry
|
||||
if err := rows.Scan(&e.ID, &e.AssetID, &e.Provider, &e.Service, &e.ProviderHost,
|
||||
&e.Kind, &e.Name, &e.Size, &e.Updated, &e.ContentType, &e.IndexedAt); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var total int
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(*) FROM asset_entries e JOIN cloud_assets a ON a.id = e.asset_id
|
||||
WHERE ($1 = '' OR e.name ILIKE '%' || $1 || '%' OR a.provider_hostname ILIKE '%' || $1 || '%')`,
|
||||
query).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// CountAssets returns the number of assets matching the filter (for pagination).
|
||||
func (s *Store) CountAssets(ctx context.Context, f AssetFilter) (int, error) {
|
||||
var n int
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(*) FROM cloud_assets
|
||||
WHERE ($1 = '' OR provider = $1)
|
||||
AND ($2 = '' OR service = $2)
|
||||
AND ($3 = '' OR exposure_state = $3)
|
||||
AND ($4 = '' OR provider_hostname ILIKE '%' || $4 || '%' OR hostname ILIKE '%' || $4 || '%' OR resource_identifier ILIKE '%' || $4 || '%')`,
|
||||
f.Provider, f.Service, f.Exposure, f.Query).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// UpdateExposure records the outcome of an optional reachability probe.
|
||||
func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState, detail string) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
@@ -170,10 +299,27 @@ type AssetFilter struct {
|
||||
Service string
|
||||
Exposure string
|
||||
Query string
|
||||
Sort string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
// orderClause maps a UI sort key to a fixed ORDER BY, preventing injection.
|
||||
func orderClause(sort string) string {
|
||||
switch sort {
|
||||
case "first_seen":
|
||||
return "first_seen DESC"
|
||||
case "provider":
|
||||
return "provider ASC, service ASC, provider_hostname ASC"
|
||||
case "hostname":
|
||||
return "provider_hostname ASC"
|
||||
case "exposure":
|
||||
return "CASE exposure_state WHEN 'public_read' THEN 0 WHEN 'reachable' THEN 1 WHEN 'auth_required' THEN 2 WHEN 'not_found' THEN 3 WHEN 'private' THEN 4 ELSE 5 END, last_seen DESC"
|
||||
default:
|
||||
return "last_seen DESC"
|
||||
}
|
||||
}
|
||||
|
||||
// ListAssets returns assets matching the filter, newest first.
|
||||
func (s *Store) ListAssets(ctx context.Context, f AssetFilter) ([]Asset, error) {
|
||||
if f.Limit <= 0 || f.Limit > 500 {
|
||||
@@ -187,7 +333,7 @@ func (s *Store) ListAssets(ctx context.Context, f AssetFilter) ([]Asset, error)
|
||||
AND ($2 = '' OR service = $2)
|
||||
AND ($3 = '' OR exposure_state = $3)
|
||||
AND ($4 = '' OR provider_hostname ILIKE '%' || $4 || '%' OR hostname ILIKE '%' || $4 || '%' OR resource_identifier ILIKE '%' || $4 || '%')
|
||||
ORDER BY last_seen DESC
|
||||
ORDER BY `+orderClause(f.Sort)+`
|
||||
LIMIT $5 OFFSET $6`,
|
||||
f.Provider, f.Service, f.Exposure, f.Query, f.Limit, f.Offset)
|
||||
if err != nil {
|
||||
@@ -217,6 +363,8 @@ type Count struct {
|
||||
type Stats struct {
|
||||
Total int64 `json:"total"`
|
||||
Readable int64 `json:"readable"`
|
||||
Reachable int64 `json:"reachable"`
|
||||
Entries int64 `json:"entries"`
|
||||
ByProvider []Count `json:"by_provider"`
|
||||
ByService []Count `json:"by_service"`
|
||||
ByExposure []Count `json:"by_exposure"`
|
||||
@@ -224,7 +372,14 @@ type Stats struct {
|
||||
|
||||
func (s *Store) Stats(ctx context.Context) (Stats, error) {
|
||||
var st Stats
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*), count(*) FILTER (WHERE exposure_state = 'public_read') FROM cloud_assets`).Scan(&st.Total, &st.Readable); err != nil {
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(*),
|
||||
count(*) FILTER (WHERE exposure_state = 'public_read'),
|
||||
count(*) FILTER (WHERE exposure_state = 'reachable')
|
||||
FROM cloud_assets`).Scan(&st.Total, &st.Readable, &st.Reachable); err != nil {
|
||||
return st, err
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*) FROM asset_entries`).Scan(&st.Entries); err != nil {
|
||||
return st, err
|
||||
}
|
||||
group := func(column string) ([]Count, error) {
|
||||
|
||||
@@ -66,7 +66,21 @@ func TestIntegration(t *testing.T) {
|
||||
if _, err := s.ListAssets(ctx, AssetFilter{Provider: "aws", Limit: 10}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.CountAssets(ctx, AssetFilter{Provider: "aws"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Stats(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := s.ReplaceEntries(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, []model.Entry{
|
||||
{Kind: "object", Name: "it/index/a.txt", Size: 10},
|
||||
{Kind: "prefix", Name: "it/index/"},
|
||||
})
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("ReplaceEntries = %d, %v", n, err)
|
||||
}
|
||||
entries, total, err := s.SearchEntries(ctx, "index/a", 10, 0)
|
||||
if err != nil || total != 1 || len(entries) != 1 || entries[0].ProviderHost != f.Hostname {
|
||||
t.Fatalf("SearchEntries = %+v total=%d err=%v", entries, total, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user