parallel backfill, persist scanned exposure, random-sample DNS queue
- scanner.run now persists exposure_state/detail/capabilities for scanned assets (previously only the index was written). - Backfill runs a bounded worker pool (16) so it verifies reachable assets without saturating the DB pool; DB_MAX_CONNS raised for the live test. - CNAME enrichment queue is now a bounded random sample instead of a FIFO. The FIFO filled with the first N hostnames seen at startup and dropped the rest, so CNAME discovery only ever saw stale hostnames; random replacement gives representative coverage at bounded memory (same fix as the prober). Verified live: - all four known-public resources scan + index (DO, AWS s3-<region>, GCS, Azure container path), 38-1000 objects each. - backfill of every CT-discovered 'reachable' asset (725) and 'auth_required' (12) completed with 0 listable: those are MRAP/non-bucket 400s and list-denied buckets, confirming the checks are actually running.
This commit is contained in:
1 parent
62a7d2eb2e
commit
171c9b37eb
6 files changed
+196
-57
No files matched your search
+23
-6
@@ -8,9 +8,12 @@ import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/scan"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
@@ -111,19 +114,33 @@ func New(s Store, cfg Config) http.Handler {
|
||||
}
|
||||
writeJSON(w, cfg.Scanner.Scan(r.Context(), target))
|
||||
})
|
||||
var backfillRunning atomic.Bool
|
||||
mux.HandleFunc("/api/reprobe", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "POST only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
q := r.URL.Query()
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
rep, err := cfg.Scanner.Backfill(r.Context(), strings.TrimSpace(q.Get("exposure")), limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
if !backfillRunning.CompareAndSwap(false, true) {
|
||||
http.Error(w, "backfill already running", http.StatusConflict)
|
||||
return
|
||||
}
|
||||
writeJSON(w, rep)
|
||||
q := r.URL.Query()
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
exposure := strings.TrimSpace(q.Get("exposure"))
|
||||
go func() {
|
||||
defer backfillRunning.Store(false)
|
||||
start := time.Now()
|
||||
rep, err := cfg.Scanner.Backfill(context.Background(), exposure, limit)
|
||||
if err != nil {
|
||||
slog.Warn("backfill failed", "event", "backfill", "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("backfill complete", "event", "backfill", "exposure", exposure,
|
||||
"probed", rep.Probed, "readable", rep.Readable, "listable", rep.Listable,
|
||||
"indexed", rep.Indexed, "errors", rep.Errors, "took", time.Since(start).String())
|
||||
}()
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
writeJSON(w, map[string]any{"started": true, "exposure": exposure, "limit": limit})
|
||||
})
|
||||
}
|
||||
mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
Reference in new issue
Block a user