provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
This commit is contained in:
1 parent
a366905a43
commit
62a7d2eb2e
23 files changed
+936
-165
No files matched your search
+14
-4
@@ -26,6 +26,7 @@ import (
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/probe"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/scan"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/webui"
|
||||
)
|
||||
@@ -128,6 +129,14 @@ func run() error {
|
||||
Metrics: m,
|
||||
}
|
||||
|
||||
// One indexer is shared by the CT prober and the on-demand scanner.
|
||||
indexer := index.New(cfg.IndexMaxObjects, 2*cfg.ProbeTimeout)
|
||||
|
||||
// Scanner prober: a separate, never-started Prober used synchronously by the
|
||||
// on-demand scan endpoint (Probe is stateless).
|
||||
scanProber := probe.New(probe.Config{Workers: 1, Timeout: cfg.ProbeTimeout, Deep: cfg.ProbeDeep, Metrics: m})
|
||||
scanner := &scan.Scanner{Prober: scanProber, Indexer: indexer, Enricher: enricher, Store: st}
|
||||
|
||||
var prober *probe.Prober
|
||||
if cfg.ActiveProbing {
|
||||
pcfg := probe.Config{
|
||||
@@ -136,12 +145,13 @@ func run() error {
|
||||
Queue: cfg.ProbeQueueSize,
|
||||
Deep: cfg.ProbeDeep,
|
||||
Metrics: m,
|
||||
Update: func(uctx context.Context, c model.Candidate, state model.ExposureState, detail string) {
|
||||
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state, detail)
|
||||
Update: func(uctx context.Context, c model.Candidate, r probe.Result) {
|
||||
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname,
|
||||
r.Exposure, r.Detail, r.Listable, r.ObjectReadable)
|
||||
},
|
||||
}
|
||||
if cfg.IndexEnabled {
|
||||
pcfg.Indexer = index.New(cfg.IndexMaxObjects, 2*cfg.ProbeTimeout)
|
||||
pcfg.Indexer = indexer
|
||||
pcfg.OnIndex = func(uctx context.Context, c model.Candidate, entries []model.Entry) {
|
||||
if n, err := st.ReplaceEntries(uctx, c.Provider.String(), c.Service.String(), c.Hostname, entries); err != nil {
|
||||
slog.Warn("index write failed", "event", "index_error", "host", c.Hostname, "error", err)
|
||||
@@ -197,7 +207,7 @@ func run() error {
|
||||
monitorDone <- monitor.Run(ctx, monitorCfg)
|
||||
}()
|
||||
|
||||
ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass})
|
||||
ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass, Scanner: scanner})
|
||||
httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady)
|
||||
go logStats(ctx, m, p, st, resolver, prober, cfg.StatsInterval)
|
||||
|
||||
|
||||
Reference in new issue
Block a user