add read-only web UI and per-vendor readability prober
Web UI (internal/webui): - dependency-free embedded HTML/JS served on the loopback listener - /api/assets (provider/service/exposure/q filters) and /api/stats - optional HTTP basic auth via UI_USER/UI_PASS Prober (internal/probe): - vendor-specific, unauthenticated, non-destructive checks: S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list, Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for database endpoints (never reported readable, no auth attempted) - exposure_detail records the factual reason (status code, region, tcp port) - bounded random-sample queue instead of FIFO: one high-volume hostname template (mrap) no longer starves every other asset type - schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER) Live: stable ~450MiB under 1GiB, no restarts, UI and probe states (private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
StasonJatham committed
1 parent
9710efec1a
commit
8fce55f8c9
13 files changed
+843
-72
No files matched your search
@@ -0,0 +1,123 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>ctcloud · cloud assets</title>
|
||||
<style>
|
||||
:root { --bg:#0f1115; --panel:#171a21; --line:#262b36; --fg:#e7e9ee; --mut:#9aa4b2; --accent:#5b9dff; }
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; font:14px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; background:var(--bg); color:var(--fg); }
|
||||
header { padding:16px 20px; border-bottom:1px solid var(--line); display:flex; align-items:baseline; gap:12px; flex-wrap:wrap; }
|
||||
h1 { font-size:16px; margin:0; letter-spacing:.5px; }
|
||||
header .mut { color:var(--mut); }
|
||||
main { padding:20px; max-width:1400px; margin:0 auto; }
|
||||
.cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:20px; }
|
||||
.card { background:var(--panel); border:1px solid var(--line); border-radius:8px; padding:12px 14px; }
|
||||
.card .n { font-size:22px; font-weight:600; }
|
||||
.card .l { color:var(--mut); font-size:12px; text-transform:uppercase; letter-spacing:.5px; }
|
||||
.filters { display:flex; gap:8px; flex-wrap:wrap; margin-bottom:14px; }
|
||||
input,select,button { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:6px; padding:7px 10px; font:inherit; }
|
||||
input { min-width:220px; }
|
||||
button { cursor:pointer; }
|
||||
button:hover { border-color:var(--accent); }
|
||||
table { width:100%; border-collapse:collapse; font-size:13px; }
|
||||
th,td { text-align:left; padding:7px 10px; border-bottom:1px solid var(--line); vertical-align:top; }
|
||||
th { color:var(--mut); font-weight:500; position:sticky; top:0; background:var(--bg); }
|
||||
tr:hover td { background:#141821; }
|
||||
.pill { display:inline-block; padding:1px 7px; border-radius:999px; font-size:11px; border:1px solid var(--line); }
|
||||
.public_read { color:#0b1; border-color:#0b1; }
|
||||
.auth_required { color:#fd7; border-color:#fd7; }
|
||||
.not_found, .private { color:#f77; border-color:#f77; }
|
||||
.reachable { color:var(--accent); border-color:var(--accent); }
|
||||
.unknown { color:var(--mut); }
|
||||
.detail { color:var(--mut); font-size:11px; }
|
||||
a { color:var(--accent); text-decoration:none; }
|
||||
footer { color:var(--mut); padding:16px 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>ctcloud</h1>
|
||||
<span class="mut">cloud/database resources discovered in Certificate Transparency</span>
|
||||
<span style="margin-left:auto" class="mut" id="updated">—</span>
|
||||
</header>
|
||||
<main>
|
||||
<div class="cards" id="cards"></div>
|
||||
<div class="filters">
|
||||
<input id="q" placeholder="search hostname / resource">
|
||||
<select id="provider" data-all="all providers"><option value="">all providers</option></select>
|
||||
<select id="service" data-all="all services"><option value="">all services</option></select>
|
||||
<select id="exposure" data-all="all exposure"><option value="">all exposure</option></select>
|
||||
<button id="refresh">refresh</button>
|
||||
</div>
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>provider</th><th>service</th><th>resource</th><th>region</th>
|
||||
<th>hostname</th><th>hostname (CT)</th><th>exposure</th><th>detail</th><th>last seen</th>
|
||||
</tr></thead>
|
||||
<tbody id="rows"></tbody>
|
||||
</table>
|
||||
</main>
|
||||
<footer>
|
||||
Detected endpoints are not automatically publicly accessible.
|
||||
<span class="public_read">public_read</span> means an anonymous read succeeded at probe time.
|
||||
<span class="reachable">reachable</span> on a database means the port was open; no authentication was attempted.
|
||||
</footer>
|
||||
<script>
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const fill = (sel, values) => {
|
||||
const cur = sel.value;
|
||||
sel.innerHTML = '<option value="">' + sel.dataset.all + '</option>' +
|
||||
values.map(v => `<option value="${v}">${v}</option>`).join('');
|
||||
sel.value = cur;
|
||||
};
|
||||
function pill(v) { return `<span class="pill ${v}">${v || 'unknown'}</span>`; }
|
||||
function ago(ts) {
|
||||
if (!ts) return '—';
|
||||
const s = Math.max(0, (Date.now() - new Date(ts)) / 1000);
|
||||
if (s < 60) return Math.floor(s) + 's ago';
|
||||
if (s < 3600) return Math.floor(s/60) + 'm ago';
|
||||
if (s < 86400) return Math.floor(s/3600) + 'h ago';
|
||||
return Math.floor(s/86400) + 'd ago';
|
||||
}
|
||||
async function loadStats() {
|
||||
const r = await fetch('api/stats'); if (!r.ok) return;
|
||||
const s = await r.json();
|
||||
$('cards').innerHTML = [
|
||||
['total assets', s.total], ['public read', s.readable],
|
||||
['providers', (s.by_provider||[]).length], ['services', (s.by_service||[]).length]
|
||||
].map(([l,n]) => `<div class="card"><div class="n">${n??0}</div><div class="l">${l}</div></div>`).join('');
|
||||
if (!$('provider').dataset.filled) {
|
||||
fill($('provider'), (s.by_provider||[]).map(x=>x.label));
|
||||
fill($('service'), (s.by_service||[]).map(x=>x.label));
|
||||
fill($('exposure'), (s.by_exposure||[]).map(x=>x.label));
|
||||
['provider','service','exposure'].forEach(id => $(id).dataset.filled = '1');
|
||||
}
|
||||
}
|
||||
async function loadAssets() {
|
||||
const p = new URLSearchParams({
|
||||
q: $('q').value, provider: $('provider').value,
|
||||
service: $('service').value, exposure: $('exposure').value, limit: 200
|
||||
});
|
||||
const r = await fetch('api/assets?' + p); if (!r.ok) return;
|
||||
const rows = await r.json() || [];
|
||||
$('rows').innerHTML = rows.length ? rows.map(a => `<tr>
|
||||
<td>${a.provider}</td><td>${a.service}</td>
|
||||
<td>${a.resource_identifier||''}</td><td>${a.region||''}</td>
|
||||
<td>${a.provider_hostname}</td><td>${a.hostname}</td>
|
||||
<td>${pill(a.exposure_state)}</td>
|
||||
<td class="detail">${a.exposure_detail||''}</td>
|
||||
<td class="detail">${ago(a.last_seen)}</td>
|
||||
</tr>`).join('') : '<tr><td colspan="9" class="detail">no assets yet</td></tr>';
|
||||
$('updated').textContent = 'updated ' + new Date().toLocaleTimeString();
|
||||
}
|
||||
async function refresh() { await Promise.all([loadStats(), loadAssets()]); }
|
||||
$('refresh').onclick = refresh;
|
||||
$('q').addEventListener('keydown', e => { if (e.key === 'Enter') loadAssets(); setTimeout(loadAssets, 0); });
|
||||
['provider','service','exposure'].forEach(id => $(id).addEventListener('change', loadAssets));
|
||||
refresh();
|
||||
setInterval(refresh, 15000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in new issue
Block a user