provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
This commit is contained in:
1 parent
a366905a43
commit
62a7d2eb2e
23 files changed
+936
-165
No files matched your search
@@ -80,6 +80,16 @@ try{var t=localStorage.getItem('bucketeer-theme')||(matchMedia('(prefers-color-s
|
||||
.sort{margin-left:auto;display:flex;align-items:center;gap:8px;color:var(--text-secondary);font:400 14px/20px var(--font)}
|
||||
select{height:40px;border:1px solid var(--border-strong);border-radius:9999px;background:var(--canvas);
|
||||
color:var(--text-primary);font:400 14px/20px var(--font);padding:0 12px}
|
||||
.scan{display:flex;gap:8px;margin-top:16px}
|
||||
.scan input{flex:1;height:40px;border:1px solid var(--border-strong);border-radius:9999px;background:var(--surface-control);
|
||||
color:var(--text-primary);font:400 14px/20px var(--font);padding:0 16px}
|
||||
.scan input:focus-visible{outline:2px solid var(--focus);outline-offset:3px;background:var(--canvas)}
|
||||
.scan button{min-height:40px;padding:0 20px;border-radius:9999px;border:1px solid var(--primary);
|
||||
background:var(--primary);color:var(--on-primary);font:500 14px/20px var(--font);cursor:pointer}
|
||||
.notice{margin-top:16px;padding:12px;border-radius:8px;background:var(--surface-subtle);border:1px solid var(--border)}
|
||||
.notice-ok{background:var(--ok-bg);border-color:transparent}
|
||||
.samples{margin-top:8px;border-top:1px solid var(--border);padding-top:8px}
|
||||
.sample{color:var(--text-secondary);font-size:13px;overflow-wrap:anywhere}
|
||||
.summary{display:flex;gap:16px;flex-wrap:wrap;margin-top:24px;color:var(--text-muted);font:400 14px/20px var(--font)}
|
||||
.summary strong{color:var(--text-primary);font-weight:500}
|
||||
|
||||
@@ -145,6 +155,14 @@ try{var t=localStorage.getItem('bucketeer-theme')||(matchMedia('(prefers-color-s
|
||||
<button class="clear" id="clear" type="button" aria-label="Clear search">×</button>
|
||||
</div>
|
||||
|
||||
<form class="scan" id="scanform">
|
||||
<label class="visually-hidden" for="scan">Scan a host or URL</label>
|
||||
<input id="scan" type="text" autocomplete="off" spellcheck="false"
|
||||
placeholder="Scan a host or URL (bucket, Azure container path, custom domain)…">
|
||||
<button type="submit">Scan</button>
|
||||
</form>
|
||||
<div id="scanout" hidden></div>
|
||||
|
||||
<div class="controls" id="asset-controls">
|
||||
<div class="filters" id="filters" role="group" aria-label="Filter by exposure">
|
||||
<button class="chip" data-exposure="" aria-pressed="true">All</button>
|
||||
@@ -256,6 +274,7 @@ function assetRow(a) {
|
||||
${a.exposure_detail ? `<p class="result-description">${esc(a.exposure_detail)}</p>` : ''}
|
||||
<div class="badge-group">
|
||||
${badge(a.provider, 'badge-cap')}${badge(a.service, 'badge-size')}
|
||||
${a.listable ? badge('listable', 'badge-ok') : (a.object_readable ? badge('object readable', 'badge-ok') : '')}
|
||||
${a.resource_identifier ? badge(a.resource_identifier) : ''}
|
||||
${a.public_semantic && a.public_semantic !== 'none' ? badge(a.public_semantic) : ''}
|
||||
</div>
|
||||
@@ -366,6 +385,40 @@ function syncURL() {
|
||||
$('asset-controls').hidden = state.view !== 'assets';
|
||||
$('q').placeholder = state.view === 'assets' ? 'Search hostname, bucket, database…' : 'Search indexed contents…';
|
||||
}
|
||||
function scanResult(r) {
|
||||
if (r.error) return `<div class="error">${esc(r.host || r.target)}: ${esc(r.error)}</div>`;
|
||||
const samples = (r.sample || []).map((s) => `<div class="mono sample">${esc(s)}</div>`).join('');
|
||||
const chain = r.cname_chain && r.cname_chain.length > 1
|
||||
? `<div class="result-meta"><span>chain: ${r.cname_chain.map(esc).join(' → ')}</span></div>` : '';
|
||||
return `<div class="notice ${r.exposure === 'public_read' ? 'notice-ok' : ''}">
|
||||
<div class="result-head">
|
||||
<span class="result-title mono">${esc(r.host)}</span>${exposureBadge(r.exposure)}
|
||||
</div>
|
||||
${r.detail ? `<p class="result-description">${esc(r.detail)}</p>` : ''}
|
||||
<div class="badge-group">${badge(r.provider, 'badge-cap')}${badge(r.service, 'badge-size')}${badge((r.entries || 0) + ' entries')}</div>
|
||||
${chain}${samples ? `<div class="samples">${samples}</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
$('scanform').addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
const target = $('scan').value.trim();
|
||||
if (!target) return;
|
||||
const out = $('scanout');
|
||||
out.hidden = false;
|
||||
out.innerHTML = '<div class="notice"><div class="skeleton w40"></div></div>';
|
||||
try {
|
||||
const resp = await fetch('api/scan', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ target }),
|
||||
});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status);
|
||||
const rows = await resp.json() || [];
|
||||
out.innerHTML = rows.map(scanResult).join('');
|
||||
if (rows.some((r) => r.entries > 0 || r.exposure === 'public_read')) reload(false);
|
||||
} catch (err) {
|
||||
out.innerHTML = `<div class="error">Scan failed: ${esc(err.message)}</div>`;
|
||||
}
|
||||
});
|
||||
|
||||
function setView(v) { state.view = v; syncURL(); reload(true); }
|
||||
|
||||
$('nav-assets').onclick = (e) => { e.preventDefault(); setView('assets'); };
|
||||
|
||||
+44
-2
@@ -12,6 +12,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/scan"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
)
|
||||
|
||||
@@ -29,9 +30,17 @@ type Store interface {
|
||||
Stats(ctx context.Context) (store.Stats, error)
|
||||
}
|
||||
|
||||
// Scanner probes and indexes a user-supplied host/URL on demand, and can
|
||||
// re-probe already-discovered assets.
|
||||
type Scanner interface {
|
||||
Scan(ctx context.Context, target string) []scan.Result
|
||||
Backfill(ctx context.Context, exposure string, limit int) (scan.BackfillReport, error)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
User string // optional HTTP basic auth username
|
||||
Pass string // optional HTTP basic auth password
|
||||
User string // optional HTTP basic auth username
|
||||
Pass string // optional HTTP basic auth password
|
||||
Scanner Scanner
|
||||
}
|
||||
|
||||
// New returns the UI handler. If Config.User is set, all routes require basic auth.
|
||||
@@ -84,6 +93,39 @@ func New(s Store, cfg Config) http.Handler {
|
||||
}
|
||||
writeJSON(w, map[string]any{"entries": entries, "total": total, "limit": limit, "offset": offset})
|
||||
})
|
||||
if cfg.Scanner != nil {
|
||||
mux.HandleFunc("/api/scan", func(w http.ResponseWriter, r *http.Request) {
|
||||
target := strings.TrimSpace(r.URL.Query().Get("target"))
|
||||
if r.Method == http.MethodPost {
|
||||
var body struct {
|
||||
Target string `json:"target"`
|
||||
}
|
||||
_ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body)
|
||||
if body.Target != "" {
|
||||
target = strings.TrimSpace(body.Target)
|
||||
}
|
||||
}
|
||||
if target == "" || len(target) > 512 {
|
||||
http.Error(w, "target required (<=512 chars)", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
writeJSON(w, cfg.Scanner.Scan(r.Context(), target))
|
||||
})
|
||||
mux.HandleFunc("/api/reprobe", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "POST only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
q := r.URL.Query()
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
rep, err := cfg.Scanner.Backfill(r.Context(), strings.TrimSpace(q.Get("exposure")), limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, rep)
|
||||
})
|
||||
}
|
||||
mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) {
|
||||
st, err := s.Stats(r.Context())
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user