StasonJatham
a366905a43
Logging: - structured slog with LOG_FORMAT=json|text and a stable key vocabulary - periodic stats record with queue depths, db pool, goroutines, memory, dns/probe - no per-certificate logging (hot path is counters only) UI (Bucketeer branding, DESIGN.md): - light+dark themes (OS default, persisted header toggle, no first-paint flash) - official mascot: lossless WebP light/dark logos + favicon, embedded and served with cache headers (1.2MB source PNGs -> ~27/22KB webp) - Assets/Contents views, pagination (offset) and native virtual scrolling (content-visibility) so large result sets do not blow up the DOM - search with debounce/abort + URL state, exposure chips, sort, summary counts Readability + indexing: - internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch (metadata only: keys/sizes/dates; never object contents) - asset_entries table + store write/search; readable resources are indexed and searchable in the UI - deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING, PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune HTTP reads - Azure account-level listing fixed: 403 is reported reachable, not auth_required, since anonymous access is per-container DNS: - curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at once broke the container resolver) - random healthy selection + failover + per-server cooldown Other: prober Close fixes a shutdown hang; config/README updated.
80 lines
3.6 KiB
SQL
80 lines
3.6 KiB
SQL
-- ctcloud schema. Applied idempotently at startup; there is no external
|
|
-- migration tooling so the container needs no writable filesystem.
|
|
|
|
CREATE TABLE IF NOT EXISTS cloud_assets (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
provider TEXT NOT NULL,
|
|
service TEXT NOT NULL,
|
|
hostname TEXT NOT NULL, -- original CT hostname (or CNAME source)
|
|
provider_hostname TEXT NOT NULL, -- hostname that matched the provider
|
|
resource_identifier TEXT NOT NULL DEFAULT '',
|
|
region TEXT NOT NULL DEFAULT '',
|
|
resource_specific BOOLEAN NOT NULL DEFAULT TRUE,
|
|
public_semantic TEXT NOT NULL DEFAULT 'none',
|
|
exposure_state TEXT NOT NULL DEFAULT 'unknown',
|
|
first_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
last_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
UNIQUE (provider, service, provider_hostname)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS cloud_assets_last_seen_idx ON cloud_assets (last_seen);
|
|
|
|
-- Reachability probe results (added after the initial schema; ALTER keeps this
|
|
-- idempotent for existing databases).
|
|
ALTER TABLE cloud_assets ADD COLUMN IF NOT EXISTS exposure_detail TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE cloud_assets ADD COLUMN IF NOT EXISTS probed_at TIMESTAMPTZ;
|
|
|
|
CREATE TABLE IF NOT EXISTS observations (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
asset_id BIGINT NOT NULL REFERENCES cloud_assets (id) ON DELETE CASCADE,
|
|
observed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
source_hostname TEXT NOT NULL,
|
|
source_type TEXT NOT NULL,
|
|
cname_chain TEXT[] NOT NULL DEFAULT '{}',
|
|
ipv4 TEXT[] NOT NULL DEFAULT '{}',
|
|
ipv6 TEXT[] NOT NULL DEFAULT '{}',
|
|
cert_fingerprint TEXT NOT NULL DEFAULT '',
|
|
ct_log_id TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS observations_asset_idx ON observations (asset_id, observed_at DESC);
|
|
|
|
-- Metadata index of resources confirmed anonymously readable. Keys/sizes/dates
|
|
-- only; never object contents.
|
|
CREATE TABLE IF NOT EXISTS asset_entries (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
asset_id BIGINT NOT NULL REFERENCES cloud_assets (id) ON DELETE CASCADE,
|
|
kind TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
size BIGINT NOT NULL DEFAULT 0,
|
|
updated TIMESTAMPTZ,
|
|
content_type TEXT NOT NULL DEFAULT '',
|
|
etag TEXT NOT NULL DEFAULT '',
|
|
indexed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
UNIQUE (asset_id, kind, name)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS asset_entries_asset_idx ON asset_entries (asset_id);
|
|
CREATE INDEX IF NOT EXISTS asset_entries_name_idx ON asset_entries (name);
|
|
|
|
-- CT resume state. `state` is the JSON-encoded certspotter monitor.LogState
|
|
-- (download/verified Merkle positions and last verified STH). No certificate
|
|
-- contents are ever stored here.
|
|
CREATE TABLE IF NOT EXISTS ct_checkpoints (
|
|
log_id TEXT PRIMARY KEY,
|
|
state JSONB NOT NULL,
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
-- Signed tree heads downloaded but not yet verified, needed to resume safely.
|
|
-- The full STH is kept as JSONB so the certspotter structure round-trips
|
|
-- exactly; tree_size is duplicated as a typed column for cheap ordering.
|
|
CREATE TABLE IF NOT EXISTS ct_sths (
|
|
log_id TEXT NOT NULL,
|
|
sth_key TEXT NOT NULL,
|
|
tree_size BIGINT NOT NULL,
|
|
sth JSONB NOT NULL,
|
|
stored_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (log_id, sth_key)
|
|
);
|