diff --git a/bucketeer-landing.jpg b/bucketeer-landing.jpg new file mode 100644 index 0000000..99afd09 Binary files /dev/null and b/bucketeer-landing.jpg differ diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 19563fc..3755ec3 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -231,7 +231,7 @@ func (d *Discoverer) scanAzure(ctx context.Context, account string) { Provider: model.ProviderAzure, Service: model.ServiceBlobStorage, ResourceSpecific: true, Source: model.SourceDiscovery, } - d.cfg.OnOpen(ctx, c, "azure public container(s): "+strings.Join(found, ","), entries) + d.cfg.OnOpen(ctx, c, "azure anonymous listing allowed (container: "+strings.Join(found, ",")+")", entries) } // validAzureAccount mirrors Azure's naming rules: 3-24 lowercase alphanumerics. @@ -299,7 +299,7 @@ func (d *Discoverer) test(ctx context.Context, t Target) { if finalHost != "" && strings.HasSuffix(finalHost, ".amazonaws.com") { c.Hostname = finalHost } - detail := "discovered open bucket via name enumeration (" + t.Hostname + ")" + detail := "anonymous listing allowed" if d.cfg.Metrics != nil { d.cfg.Metrics.BucketOpen.Add(1) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ddbbed4..b4a2d15 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -90,7 +90,7 @@ func (m *Metrics) Handler() http.Handler { counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load()) counter("entries_indexed_total", "Metadata entries indexed from readable resources", m.EntriesIndexed.Load()) counter("bucket_candidates_total", "Candidate bucket names tested", m.BucketCandidates.Load()) - counter("bucket_open_total", "Open (listable) buckets discovered by name enumeration", m.BucketOpen.Load()) + counter("bucket_open_total", "Open (listable) buckets discovered from CT hostname labels", m.BucketOpen.Load()) for _, g := range m.gauges { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn()) } diff --git a/internal/store/state.go b/internal/store/state.go index 3be60d0..e7977d8 100644 --- a/internal/store/state.go +++ b/internal/store/state.go @@ -2,6 +2,7 @@ package store import ( "context" + "crypto/sha256" "encoding/json" "fmt" "log" @@ -35,11 +36,21 @@ func (s *Store) StoreLogState(ctx context.Context, logID monitor.LogID, state *m if err != nil { return err } + key := logID.Base64URLString() + hash := sha256.Sum256(data) + s.stateMu.Lock() + if prev, ok := s.lastState[key]; ok && prev == hash { + s.stateMu.Unlock() + return nil // unchanged; CertSpotter calls this per batch + } + s.lastState[key] = hash + s.stateMu.Unlock() + _, err = s.pool.Exec(ctx, ` INSERT INTO ct_checkpoints (log_id, state, updated_at) VALUES ($1, $2, now()) ON CONFLICT (log_id) DO UPDATE SET state = EXCLUDED.state, updated_at = now()`, - logID.Base64URLString(), data) + key, data) return err } diff --git a/internal/store/store.go b/internal/store/store.go index 0b4aaad..0b06aa3 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -5,6 +5,7 @@ package store import ( "context" "fmt" + "sync" "time" "github.com/jackc/pgx/v5" @@ -20,6 +21,9 @@ import ( type Store struct { pool *pgxpool.Pool metrics *metrics.Metrics + + stateMu sync.Mutex + lastState map[string][32]byte // per-log checkpoint hash; skip identical writes } // Open connects to PostgreSQL. The pool is bounded; the caller configures the @@ -38,7 +42,7 @@ func Open(ctx context.Context, databaseURL string, maxConns int) (*Store, error) if err != nil { return nil, err } - return &Store{pool: pool}, nil + return &Store{pool: pool, lastState: make(map[string][32]byte)}, nil } func (s *Store) SetMetrics(m *metrics.Metrics) { s.metrics = m } diff --git a/internal/webui/webui.go b/internal/webui/webui.go index b15ac96..8ee2516 100644 --- a/internal/webui/webui.go +++ b/internal/webui/webui.go @@ -12,6 +12,7 @@ import ( "net/http" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -73,14 +74,16 @@ func New(s Store, cfg Config) http.Handler { Limit: limit, Offset: offset, } - assets, err := s.ListAssets(r.Context(), f) + ctx, cancel := context.WithTimeout(r.Context(), 9*time.Second) + defer cancel() + assets, err := s.ListAssets(ctx, f) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusServiceUnavailable) return } - total, err := s.CountAssets(r.Context(), f) + total, err := s.CountAssets(ctx, f) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusServiceUnavailable) return } writeJSON(w, map[string]any{"assets": assets, "total": total, "limit": limit, "offset": offset}) @@ -89,9 +92,11 @@ func New(s Store, cfg Config) http.Handler { q := r.URL.Query() limit, _ := strconv.Atoi(q.Get("limit")) offset, _ := strconv.Atoi(q.Get("offset")) - entries, total, err := s.SearchEntries(r.Context(), strings.TrimSpace(q.Get("q")), limit, offset) + ctx, cancel := context.WithTimeout(r.Context(), 9*time.Second) + defer cancel() + entries, total, err := s.SearchEntries(ctx, strings.TrimSpace(q.Get("q")), limit, offset) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusServiceUnavailable) return } writeJSON(w, map[string]any{"entries": entries, "total": total, "limit": limit, "offset": offset}) @@ -143,12 +148,28 @@ func New(s Store, cfg Config) http.Handler { writeJSON(w, map[string]any{"started": true, "exposure": exposure, "limit": limit}) }) } + var ( + statsMu sync.Mutex + statsVal store.Stats + statsExp time.Time + ) mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) { - st, err := s.Stats(r.Context()) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + // Cached for a few seconds: the UI polls, and each computation is several + // aggregate queries that can queue behind CertSpotter checkpoint bursts. + statsMu.Lock() + defer statsMu.Unlock() + if time.Now().Before(statsExp) { + writeJSON(w, statsVal) return } + ctx, cancel := context.WithTimeout(r.Context(), 9*time.Second) + defer cancel() + st, err := s.Stats(ctx) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + statsVal, statsExp = st, time.Now().Add(10*time.Second) writeJSON(w, st) })