StasonJatham
e8f7d6bfdf
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
57 lines
2.5 KiB
SQL
57 lines
2.5 KiB
SQL
-- ctcloud schema. Applied idempotently at startup; there is no external
|
|
-- migration tooling so the container needs no writable filesystem.
|
|
|
|
CREATE TABLE IF NOT EXISTS cloud_assets (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
provider TEXT NOT NULL,
|
|
service TEXT NOT NULL,
|
|
hostname TEXT NOT NULL, -- original CT hostname (or CNAME source)
|
|
provider_hostname TEXT NOT NULL, -- hostname that matched the provider
|
|
resource_identifier TEXT NOT NULL DEFAULT '',
|
|
region TEXT NOT NULL DEFAULT '',
|
|
resource_specific BOOLEAN NOT NULL DEFAULT TRUE,
|
|
public_semantic TEXT NOT NULL DEFAULT 'none',
|
|
exposure_state TEXT NOT NULL DEFAULT 'unknown',
|
|
first_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
last_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
UNIQUE (provider, service, provider_hostname)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS cloud_assets_last_seen_idx ON cloud_assets (last_seen);
|
|
|
|
CREATE TABLE IF NOT EXISTS observations (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
asset_id BIGINT NOT NULL REFERENCES cloud_assets (id) ON DELETE CASCADE,
|
|
observed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
source_hostname TEXT NOT NULL,
|
|
source_type TEXT NOT NULL,
|
|
cname_chain TEXT[] NOT NULL DEFAULT '{}',
|
|
ipv4 TEXT[] NOT NULL DEFAULT '{}',
|
|
ipv6 TEXT[] NOT NULL DEFAULT '{}',
|
|
cert_fingerprint TEXT NOT NULL DEFAULT '',
|
|
ct_log_id TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS observations_asset_idx ON observations (asset_id, observed_at DESC);
|
|
|
|
-- CT resume state. `state` is the JSON-encoded certspotter monitor.LogState
|
|
-- (download/verified Merkle positions and last verified STH). No certificate
|
|
-- contents are ever stored here.
|
|
CREATE TABLE IF NOT EXISTS ct_checkpoints (
|
|
log_id TEXT PRIMARY KEY,
|
|
state JSONB NOT NULL,
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
-- Signed tree heads downloaded but not yet verified, needed to resume safely.
|
|
-- The full STH is kept as JSONB so the certspotter structure round-trips
|
|
-- exactly; tree_size is duplicated as a typed column for cheap ordering.
|
|
CREATE TABLE IF NOT EXISTS ct_sths (
|
|
log_id TEXT NOT NULL,
|
|
sth_key TEXT NOT NULL,
|
|
tree_size BIGINT NOT NULL,
|
|
sth JSONB NOT NULL,
|
|
stored_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (log_id, sth_key)
|
|
);
|