add read-only web UI and per-vendor readability prober
Web UI (internal/webui): - dependency-free embedded HTML/JS served on the loopback listener - /api/assets (provider/service/exposure/q filters) and /api/stats - optional HTTP basic auth via UI_USER/UI_PASS Prober (internal/probe): - vendor-specific, unauthenticated, non-destructive checks: S3/R2/Spaces ListObjectsV2, GCS JSON object list, Azure ?comp=list, Firebase /.json?shallow=true, Supabase REST, and bare TCP connect for database endpoints (never reported readable, no auth attempted) - exposure_detail records the factual reason (status code, region, tcp port) - bounded random-sample queue instead of FIFO: one high-volume hostname template (mrap) no longer starves every other asset type - schema: cloud_assets.exposure_detail + probed_at (idempotent ALTER) Live: stable ~450MiB under 1GiB, no restarts, UI and probe states (private/reachable/auth_required) flowing, 0 db errors.
This commit is contained in:
StasonJatham committed
1 parent
9710efec1a
commit
8fce55f8c9
13 files changed
+843
-72
No files matched your search
@@ -0,0 +1,123 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>ctcloud · cloud assets</title>
|
||||
<style>
|
||||
:root { --bg:#0f1115; --panel:#171a21; --line:#262b36; --fg:#e7e9ee; --mut:#9aa4b2; --accent:#5b9dff; }
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; font:14px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; background:var(--bg); color:var(--fg); }
|
||||
header { padding:16px 20px; border-bottom:1px solid var(--line); display:flex; align-items:baseline; gap:12px; flex-wrap:wrap; }
|
||||
h1 { font-size:16px; margin:0; letter-spacing:.5px; }
|
||||
header .mut { color:var(--mut); }
|
||||
main { padding:20px; max-width:1400px; margin:0 auto; }
|
||||
.cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:20px; }
|
||||
.card { background:var(--panel); border:1px solid var(--line); border-radius:8px; padding:12px 14px; }
|
||||
.card .n { font-size:22px; font-weight:600; }
|
||||
.card .l { color:var(--mut); font-size:12px; text-transform:uppercase; letter-spacing:.5px; }
|
||||
.filters { display:flex; gap:8px; flex-wrap:wrap; margin-bottom:14px; }
|
||||
input,select,button { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:6px; padding:7px 10px; font:inherit; }
|
||||
input { min-width:220px; }
|
||||
button { cursor:pointer; }
|
||||
button:hover { border-color:var(--accent); }
|
||||
table { width:100%; border-collapse:collapse; font-size:13px; }
|
||||
th,td { text-align:left; padding:7px 10px; border-bottom:1px solid var(--line); vertical-align:top; }
|
||||
th { color:var(--mut); font-weight:500; position:sticky; top:0; background:var(--bg); }
|
||||
tr:hover td { background:#141821; }
|
||||
.pill { display:inline-block; padding:1px 7px; border-radius:999px; font-size:11px; border:1px solid var(--line); }
|
||||
.public_read { color:#0b1; border-color:#0b1; }
|
||||
.auth_required { color:#fd7; border-color:#fd7; }
|
||||
.not_found, .private { color:#f77; border-color:#f77; }
|
||||
.reachable { color:var(--accent); border-color:var(--accent); }
|
||||
.unknown { color:var(--mut); }
|
||||
.detail { color:var(--mut); font-size:11px; }
|
||||
a { color:var(--accent); text-decoration:none; }
|
||||
footer { color:var(--mut); padding:16px 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>ctcloud</h1>
|
||||
<span class="mut">cloud/database resources discovered in Certificate Transparency</span>
|
||||
<span style="margin-left:auto" class="mut" id="updated">—</span>
|
||||
</header>
|
||||
<main>
|
||||
<div class="cards" id="cards"></div>
|
||||
<div class="filters">
|
||||
<input id="q" placeholder="search hostname / resource">
|
||||
<select id="provider" data-all="all providers"><option value="">all providers</option></select>
|
||||
<select id="service" data-all="all services"><option value="">all services</option></select>
|
||||
<select id="exposure" data-all="all exposure"><option value="">all exposure</option></select>
|
||||
<button id="refresh">refresh</button>
|
||||
</div>
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>provider</th><th>service</th><th>resource</th><th>region</th>
|
||||
<th>hostname</th><th>hostname (CT)</th><th>exposure</th><th>detail</th><th>last seen</th>
|
||||
</tr></thead>
|
||||
<tbody id="rows"></tbody>
|
||||
</table>
|
||||
</main>
|
||||
<footer>
|
||||
Detected endpoints are not automatically publicly accessible.
|
||||
<span class="public_read">public_read</span> means an anonymous read succeeded at probe time.
|
||||
<span class="reachable">reachable</span> on a database means the port was open; no authentication was attempted.
|
||||
</footer>
|
||||
<script>
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const fill = (sel, values) => {
|
||||
const cur = sel.value;
|
||||
sel.innerHTML = '<option value="">' + sel.dataset.all + '</option>' +
|
||||
values.map(v => `<option value="${v}">${v}</option>`).join('');
|
||||
sel.value = cur;
|
||||
};
|
||||
function pill(v) { return `<span class="pill ${v}">${v || 'unknown'}</span>`; }
|
||||
function ago(ts) {
|
||||
if (!ts) return '—';
|
||||
const s = Math.max(0, (Date.now() - new Date(ts)) / 1000);
|
||||
if (s < 60) return Math.floor(s) + 's ago';
|
||||
if (s < 3600) return Math.floor(s/60) + 'm ago';
|
||||
if (s < 86400) return Math.floor(s/3600) + 'h ago';
|
||||
return Math.floor(s/86400) + 'd ago';
|
||||
}
|
||||
async function loadStats() {
|
||||
const r = await fetch('api/stats'); if (!r.ok) return;
|
||||
const s = await r.json();
|
||||
$('cards').innerHTML = [
|
||||
['total assets', s.total], ['public read', s.readable],
|
||||
['providers', (s.by_provider||[]).length], ['services', (s.by_service||[]).length]
|
||||
].map(([l,n]) => `<div class="card"><div class="n">${n??0}</div><div class="l">${l}</div></div>`).join('');
|
||||
if (!$('provider').dataset.filled) {
|
||||
fill($('provider'), (s.by_provider||[]).map(x=>x.label));
|
||||
fill($('service'), (s.by_service||[]).map(x=>x.label));
|
||||
fill($('exposure'), (s.by_exposure||[]).map(x=>x.label));
|
||||
['provider','service','exposure'].forEach(id => $(id).dataset.filled = '1');
|
||||
}
|
||||
}
|
||||
async function loadAssets() {
|
||||
const p = new URLSearchParams({
|
||||
q: $('q').value, provider: $('provider').value,
|
||||
service: $('service').value, exposure: $('exposure').value, limit: 200
|
||||
});
|
||||
const r = await fetch('api/assets?' + p); if (!r.ok) return;
|
||||
const rows = await r.json() || [];
|
||||
$('rows').innerHTML = rows.length ? rows.map(a => `<tr>
|
||||
<td>${a.provider}</td><td>${a.service}</td>
|
||||
<td>${a.resource_identifier||''}</td><td>${a.region||''}</td>
|
||||
<td>${a.provider_hostname}</td><td>${a.hostname}</td>
|
||||
<td>${pill(a.exposure_state)}</td>
|
||||
<td class="detail">${a.exposure_detail||''}</td>
|
||||
<td class="detail">${ago(a.last_seen)}</td>
|
||||
</tr>`).join('') : '<tr><td colspan="9" class="detail">no assets yet</td></tr>';
|
||||
$('updated').textContent = 'updated ' + new Date().toLocaleTimeString();
|
||||
}
|
||||
async function refresh() { await Promise.all([loadStats(), loadAssets()]); }
|
||||
$('refresh').onclick = refresh;
|
||||
$('q').addEventListener('keydown', e => { if (e.key === 'Enter') loadAssets(); setTimeout(loadAssets, 0); });
|
||||
['provider','service','exposure'].forEach(id => $(id).addEventListener('change', loadAssets));
|
||||
refresh();
|
||||
setInterval(refresh, 15000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,90 @@
|
||||
// Package webui serves a small read-only web UI over the discovered cloud
|
||||
// assets. It is intentionally dependency-free (stdlib + embedded HTML) and
|
||||
// exposes no mutation endpoints.
|
||||
package webui
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
)
|
||||
|
||||
//go:embed index.html
|
||||
var indexHTML []byte
|
||||
|
||||
// Store is the read-only subset of the database the UI needs.
|
||||
type Store interface {
|
||||
ListAssets(ctx context.Context, f store.AssetFilter) ([]store.Asset, error)
|
||||
Stats(ctx context.Context) (store.Stats, error)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
User string // optional HTTP basic auth username
|
||||
Pass string // optional HTTP basic auth password
|
||||
}
|
||||
|
||||
// New returns the UI handler. If Config.User is set, all routes require basic auth.
|
||||
func New(s Store, cfg Config) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write(indexHTML)
|
||||
})
|
||||
mux.HandleFunc("/api/assets", func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
offset, _ := strconv.Atoi(q.Get("offset"))
|
||||
assets, err := s.ListAssets(r.Context(), store.AssetFilter{
|
||||
Provider: strings.TrimSpace(q.Get("provider")),
|
||||
Service: strings.TrimSpace(q.Get("service")),
|
||||
Exposure: strings.TrimSpace(q.Get("exposure")),
|
||||
Query: strings.TrimSpace(q.Get("q")),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, assets)
|
||||
})
|
||||
mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) {
|
||||
st, err := s.Stats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, st)
|
||||
})
|
||||
|
||||
if cfg.User != "" {
|
||||
return basicAuth(cfg.User, cfg.Pass, mux)
|
||||
}
|
||||
return mux
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func basicAuth(user, pass string, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
u, p, ok := r.BasicAuth()
|
||||
if !ok || u != user || p != pass {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="ctcloud"`)
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
lastFilter store.AssetFilter
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListAssets(ctx context.Context, fl store.AssetFilter) ([]store.Asset, error) {
|
||||
f.lastFilter = fl
|
||||
return []store.Asset{{ID: 1, Provider: "aws", Service: "s3", ProviderHost: "b.s3.amazonaws.com", Exposure: "public_read"}}, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Stats(ctx context.Context) (store.Stats, error) {
|
||||
return store.Stats{Total: 1, Readable: 1}, nil
|
||||
}
|
||||
|
||||
func TestAPI(t *testing.T) {
|
||||
fs := &fakeStore{}
|
||||
srv := httptest.NewServer(New(fs, Config{}))
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/api/assets?provider=aws&q=bucket&limit=5")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
var assets []store.Asset
|
||||
if err := json.NewDecoder(resp.Body).Decode(&assets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(assets) != 1 || fs.lastFilter.Provider != "aws" || fs.lastFilter.Query != "bucket" || fs.lastFilter.Limit != 5 {
|
||||
t.Fatalf("unexpected assets/filter: %+v %+v", assets, fs.lastFilter)
|
||||
}
|
||||
|
||||
statsResp, err := http.Get(srv.URL + "/api/stats")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer statsResp.Body.Close()
|
||||
var st store.Stats
|
||||
if err := json.NewDecoder(statsResp.Body).Decode(&st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.Total != 1 {
|
||||
t.Fatalf("stats = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicAuth(t *testing.T) {
|
||||
srv := httptest.NewServer(New(&fakeStore{}, Config{User: "u", Pass: "p"}))
|
||||
defer srv.Close()
|
||||
|
||||
resp, _ := http.Get(srv.URL + "/api/stats")
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", resp.StatusCode)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
req, _ := http.NewRequest(http.MethodGet, srv.URL+"/api/stats", nil)
|
||||
req.SetBasicAuth("u", "p")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexServed(t *testing.T) {
|
||||
srv := httptest.NewServer(New(&fakeStore{}, Config{}))
|
||||
defer srv.Close()
|
||||
resp, err := http.Get(srv.URL + "/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "text/html; charset=utf-8" {
|
||||
t.Fatalf("content-type = %q", ct)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user