ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
This commit is contained in:
StasonJatham committed
1 parent
9e78a114ac
commit
e8f7d6bfdf
118 files changed
+16473
No files matched your search
@@ -0,0 +1,165 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"software.sslmate.com/src/certspotter/cttypes"
|
||||
"software.sslmate.com/src/certspotter/loglist"
|
||||
"software.sslmate.com/src/certspotter/monitor"
|
||||
)
|
||||
|
||||
var _ monitor.StateProvider = (*Store)(nil)
|
||||
|
||||
// Prepare implements monitor.StateProvider. It applies the embedded schema so
|
||||
// the container needs no writable filesystem.
|
||||
func (s *Store) Prepare(ctx context.Context) error {
|
||||
if err := s.Migrate(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrepareLog is a no-op: checkpoint rows are created lazily by StoreLogState.
|
||||
func (s *Store) PrepareLog(ctx context.Context, logID monitor.LogID) error { return nil }
|
||||
|
||||
// StoreLogState persists the certspotter monitor position for a log.
|
||||
func (s *Store) StoreLogState(ctx context.Context, logID monitor.LogID, state *monitor.LogState) error {
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.pool.Exec(ctx, `
|
||||
INSERT INTO ct_checkpoints (log_id, state, updated_at)
|
||||
VALUES ($1, $2, now())
|
||||
ON CONFLICT (log_id) DO UPDATE SET state = EXCLUDED.state, updated_at = now()`,
|
||||
logID.Base64URLString(), data)
|
||||
return err
|
||||
}
|
||||
|
||||
// LoadLogState returns nil when no checkpoint exists for the log.
|
||||
func (s *Store) LoadLogState(ctx context.Context, logID monitor.LogID) (*monitor.LogState, error) {
|
||||
var data []byte
|
||||
err := s.pool.QueryRow(ctx, `SELECT state FROM ct_checkpoints WHERE log_id = $1`, logID.Base64URLString()).Scan(&data)
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state := new(monitor.LogState)
|
||||
if err := json.Unmarshal(data, state); err != nil {
|
||||
return nil, fmt.Errorf("corrupt checkpoint for log %s: %w", logID.Base64URLString(), err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func sthKey(sth *cttypes.SignedTreeHead) string {
|
||||
return strconv.FormatUint(sth.TreeSize, 10) + "-" +
|
||||
strconv.FormatUint(sth.Timestamp, 10) + "-" +
|
||||
sth.RootHash.Base64String()
|
||||
}
|
||||
|
||||
// StoreSTH stores an unverified signed tree head. If it already exists the
|
||||
// original stored_at is preserved.
|
||||
func (s *Store) StoreSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) (*monitor.StoredSTH, error) {
|
||||
data, err := json.Marshal(sth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var storedAt time.Time
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
INSERT INTO ct_sths (log_id, sth_key, tree_size, sth, stored_at)
|
||||
VALUES ($1, $2, $3, $4, now())
|
||||
ON CONFLICT (log_id, sth_key) DO UPDATE SET log_id = EXCLUDED.log_id
|
||||
RETURNING stored_at`,
|
||||
logID.Base64URLString(), sthKey(sth), sth.TreeSize, data).Scan(&storedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &monitor.StoredSTH{SignedTreeHead: *sth, StoredAt: storedAt}, nil
|
||||
}
|
||||
|
||||
// LoadSTHs loads all stored STHs for a log, sorted by tree size.
|
||||
func (s *Store) LoadSTHs(ctx context.Context, logID monitor.LogID) ([]*monitor.StoredSTH, error) {
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT sth, stored_at FROM ct_sths WHERE log_id = $1 ORDER BY tree_size ASC`,
|
||||
logID.Base64URLString())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*monitor.StoredSTH
|
||||
for rows.Next() {
|
||||
var (
|
||||
data []byte
|
||||
storedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(&data, &storedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sth cttypes.SignedTreeHead
|
||||
if err := json.Unmarshal(data, &sth); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, &monitor.StoredSTH{SignedTreeHead: sth, StoredAt: storedAt})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RemoveSTH deletes an STH once it has been verified against the Merkle tree.
|
||||
func (s *Store) RemoveSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) error {
|
||||
_, err := s.pool.Exec(ctx, `DELETE FROM ct_sths WHERE log_id = $1 AND sth_key = $2`,
|
||||
logID.Base64URLString(), sthKey(sth))
|
||||
return err
|
||||
}
|
||||
|
||||
// StoreIssuer and LoadIssuer are no-ops. ctcloud's identifier sink bypasses
|
||||
// certificate-chain retrieval entirely, so the issuer cache is never consulted;
|
||||
// keeping it empty also satisfies the "no certificate contents in the state
|
||||
// store" requirement. The Go runtime cache handles any incidental calls.
|
||||
func (s *Store) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// NotifyCert is never called because the identifier sink handles certificates.
|
||||
func (s *Store) NotifyCert(ctx context.Context, cert *monitor.DiscoveredCert) error { return nil }
|
||||
|
||||
// NotifyMalformedEntry counts unparseable log entries without logging per entry.
|
||||
func (s *Store) NotifyMalformedEntry(ctx context.Context, entry *monitor.LogEntry, parseError error) error {
|
||||
if s.metrics != nil {
|
||||
s.metrics.MalformedCT.Add(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NotifyHealthCheckFailure is rare (once per health-check interval at most), so
|
||||
// it is safe to log.
|
||||
func (s *Store) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info monitor.HealthCheckFailure) error {
|
||||
log.Printf("ct health check failure: %s", info.Summary())
|
||||
return nil
|
||||
}
|
||||
|
||||
// NotifyError counts transient CT errors. It deliberately does not log, to keep
|
||||
// logging off the high-frequency error path.
|
||||
func (s *Store) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error {
|
||||
if s.metrics != nil {
|
||||
s.metrics.CTErrors.Add(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetErrors returns the recent CT errors. ctcloud keeps only aggregate counters.
|
||||
func (s *Store) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// Package store owns the PostgreSQL schema, the batched findings writer and the
|
||||
// restart-safe CT checkpoint state provider.
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/metrics"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
|
||||
"tea.karlcom.de/karl/bucketeer/migrations"
|
||||
)
|
||||
|
||||
// Store wraps a pgx pool with the operations ctcloud needs.
|
||||
type Store struct {
|
||||
pool *pgxpool.Pool
|
||||
metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// Open connects to PostgreSQL. The pool is bounded; the caller configures the
|
||||
// maximum number of connections.
|
||||
func Open(ctx context.Context, databaseURL string, maxConns int) (*Store, error) {
|
||||
cfg, err := pgxpool.ParseConfig(databaseURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DATABASE_URL: %w", err)
|
||||
}
|
||||
if maxConns > 0 {
|
||||
cfg.MaxConns = int32(maxConns)
|
||||
}
|
||||
cfg.MaxConnLifetime = 30 * time.Minute
|
||||
cfg.MinConns = 1
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{pool: pool}, nil
|
||||
}
|
||||
|
||||
func (s *Store) SetMetrics(m *metrics.Metrics) { s.metrics = m }
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
|
||||
|
||||
func (s *Store) Close() { s.pool.Close() }
|
||||
|
||||
func (s *Store) Pool() *pgxpool.Pool { return s.pool }
|
||||
|
||||
// Migrate applies the embedded schema idempotently.
|
||||
func (s *Store) Migrate(ctx context.Context) error {
|
||||
for _, name := range migrations.SQL() {
|
||||
sqlBytes, err := migrations.FS.ReadFile(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, string(sqlBytes)); err != nil {
|
||||
return fmt.Errorf("applying %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const upsertFindingSQL = `
|
||||
WITH asset AS (
|
||||
INSERT INTO cloud_assets
|
||||
(provider, service, hostname, provider_hostname, resource_identifier,
|
||||
region, resource_specific, public_semantic, first_seen, last_seen)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now())
|
||||
ON CONFLICT (provider, service, provider_hostname)
|
||||
DO UPDATE SET
|
||||
last_seen = now(),
|
||||
hostname = EXCLUDED.hostname,
|
||||
resource_identifier = CASE WHEN EXCLUDED.resource_identifier = '' THEN cloud_assets.resource_identifier ELSE EXCLUDED.resource_identifier END,
|
||||
region = CASE WHEN EXCLUDED.region = '' THEN cloud_assets.region ELSE EXCLUDED.region END
|
||||
RETURNING id
|
||||
)
|
||||
INSERT INTO observations
|
||||
(asset_id, source_hostname, source_type, cname_chain, ipv4, ipv6)
|
||||
SELECT asset.id, $9, $10, $11, $12, $13 FROM asset`
|
||||
|
||||
// WriteFindings writes one batch of findings in a single round trip. Asset rows
|
||||
// are upserted and an observation is appended for each finding.
|
||||
func (s *Store) WriteFindings(ctx context.Context, batch []pipeline.Finding) error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
var lastErr error
|
||||
// ponytail: three quick attempts; the pipeline keeps running and the
|
||||
// bounded DB queue sheds load if PostgreSQL stays down.
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if err := s.writeBatch(ctx, batch); err != nil {
|
||||
lastErr = err
|
||||
if ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
backoff := time.Duration(1<<attempt) * 100 * time.Millisecond
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (s *Store) writeBatch(ctx context.Context, batch []pipeline.Finding) error {
|
||||
b := &pgx.Batch{}
|
||||
for i := range batch {
|
||||
f := &batch[i]
|
||||
b.Queue(upsertFindingSQL,
|
||||
f.Provider.String(), f.Service.String(), f.Original, f.Hostname,
|
||||
f.Resource, f.Region, f.ResourceSpecific, f.PublicSemantic.String(),
|
||||
f.Original, f.Source.String(),
|
||||
nonNil(f.CNAMEChain), nonNil(f.IPv4), nonNil(f.IPv6),
|
||||
)
|
||||
}
|
||||
br := s.pool.SendBatch(ctx, b)
|
||||
defer br.Close()
|
||||
for range batch {
|
||||
if _, err := br.Exec(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func nonNil(s []string) []string {
|
||||
if s == nil {
|
||||
return []string{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// UpdateExposure records the outcome of an optional reachability probe.
|
||||
func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE cloud_assets SET exposure_state = $4 WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
|
||||
provider, service, providerHostname, state.String())
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"software.sslmate.com/src/certspotter/cttypes"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
|
||||
)
|
||||
|
||||
func TestSTHKeyStableAndUnique(t *testing.T) {
|
||||
a := &cttypes.SignedTreeHead{TreeSize: 10, Timestamp: 100}
|
||||
if sthKey(a) != sthKey(a) {
|
||||
t.Fatal("sthKey is not deterministic")
|
||||
}
|
||||
b := &cttypes.SignedTreeHead{TreeSize: 11, Timestamp: 100}
|
||||
if sthKey(a) == sthKey(b) {
|
||||
t.Fatal("sthKey collides for different tree sizes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonNil(t *testing.T) {
|
||||
if got := nonNil(nil); got == nil || len(got) != 0 {
|
||||
t.Fatalf("nonNil(nil) = %#v", got)
|
||||
}
|
||||
in := []string{"a"}
|
||||
if got := nonNil(in); len(got) != 1 || got[0] != "a" {
|
||||
t.Fatalf("nonNil changed slice: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIntegration exercises the real schema when DATABASE_URL is set.
|
||||
func TestIntegration(t *testing.T) {
|
||||
url := os.Getenv("DATABASE_URL")
|
||||
if url == "" {
|
||||
t.Skip("DATABASE_URL not set; skipping integration test")
|
||||
}
|
||||
ctx := t.Context()
|
||||
s, err := Open(ctx, url, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := pipeline.Finding{
|
||||
Candidate: model.Candidate{
|
||||
Hostname: "it-bucket.s3.eu-central-1.amazonaws.com",
|
||||
Original: "it-bucket.s3.eu-central-1.amazonaws.com",
|
||||
Provider: model.ProviderAWS,
|
||||
Service: model.ServiceS3,
|
||||
Source: model.SourceDirectCT,
|
||||
},
|
||||
IPv4: []string{"1.2.3.4"},
|
||||
}
|
||||
if err := s.WriteFindings(ctx, []pipeline.Finding{f}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpdateExposure(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, model.ExposureReachable); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user