ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
This commit is contained in:
StasonJatham committed
1 parent
9e78a114ac
commit
e8f7d6bfdf
118 files changed
+16473
No files matched your search
@@ -0,0 +1,234 @@
|
||||
// Command ctcloud continuously consumes Certificate Transparency logs through
|
||||
// embedded CertSpotter internals, classifies cloud/database/storage resources
|
||||
// in-process and persists only useful, enriched findings to PostgreSQL.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"software.sslmate.com/src/certspotter/monitor"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/config"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/dns"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/metrics"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/probe"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/store"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
slog.Error("fatal", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func levelFromString(s string) slog.Level {
|
||||
switch s {
|
||||
case "debug":
|
||||
return slog.LevelDebug
|
||||
case "warn":
|
||||
return slog.LevelWarn
|
||||
case "error":
|
||||
return slog.LevelError
|
||||
default:
|
||||
return slog.LevelInfo
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: levelFromString(cfg.LogLevel)})))
|
||||
if strings.Contains(cfg.DatabaseURL, "sslmode=disable") {
|
||||
slog.Warn("DATABASE_URL has sslmode=disable; only use this on a private network")
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
m := metrics.New()
|
||||
|
||||
st, err := store.Open(ctx, cfg.DatabaseURL, cfg.DBMaxConns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
st.SetMetrics(m)
|
||||
if err := st.Migrate(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("database ready")
|
||||
|
||||
enricher := &dns.Enricher{
|
||||
Resolver: dns.NewServerPool(cfg.DNSServers),
|
||||
Cache: dns.NewTTLCache[dns.Result](cfg.DNSCacheSize, 64),
|
||||
PositiveTTL: cfg.DNSPositiveTTL,
|
||||
NegativeTTL: cfg.DNSNegativeTTL,
|
||||
Timeout: cfg.DNSTimeout,
|
||||
MaxDepth: cfg.DNSMaxCNAMEdepth,
|
||||
Metrics: m,
|
||||
}
|
||||
|
||||
var prober *probe.Prober
|
||||
if cfg.ActiveProbing {
|
||||
prober = probe.New(probe.Config{
|
||||
Workers: cfg.ProbeWorkers,
|
||||
Timeout: cfg.ProbeTimeout,
|
||||
Metrics: m,
|
||||
Update: func(uctx context.Context, c model.Candidate, state model.ExposureState) {
|
||||
_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state)
|
||||
},
|
||||
})
|
||||
prober.Start(ctx)
|
||||
slog.Info("active probing enabled")
|
||||
}
|
||||
|
||||
write := func(wctx context.Context, batch []pipeline.Finding) error {
|
||||
err := st.WriteFindings(wctx, batch)
|
||||
if err == nil && prober != nil {
|
||||
for i := range batch {
|
||||
prober.Submit(batch[i].Candidate)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
p := pipeline.New(pipeline.Config{
|
||||
DirectQueueSize: cfg.DirectQueueSize,
|
||||
DNSQueueSize: cfg.DNSQueueSize,
|
||||
DBQueueSize: cfg.DBQueueSize,
|
||||
DNSWorkers: cfg.DNSWorkers,
|
||||
BatchSize: cfg.DBBatchSize,
|
||||
FlushInterval: cfg.DBFlushInterval,
|
||||
DedupTTL: cfg.DedupeTTL,
|
||||
Enricher: enricher,
|
||||
Write: write,
|
||||
Metrics: m,
|
||||
})
|
||||
p.Start(ctx)
|
||||
|
||||
var monitorReady atomic.Bool
|
||||
monitorCfg := &monitor.Config{
|
||||
LogListSource: cfg.LogListSource,
|
||||
State: st,
|
||||
StartAtEnd: cfg.StartAtEnd,
|
||||
Verbose: cfg.LogLevel == "debug",
|
||||
HealthCheckInterval: 24 * time.Hour,
|
||||
IdentifierSink: p,
|
||||
}
|
||||
|
||||
monitorDone := make(chan error, 1)
|
||||
go func() {
|
||||
monitorReady.Store(true)
|
||||
monitorDone <- monitor.Run(ctx, monitorCfg)
|
||||
}()
|
||||
|
||||
httpSrv := startHTTP(cfg.MetricsAddr, m, st, &monitorReady)
|
||||
go logStats(ctx, m, cfg.StatsInterval)
|
||||
|
||||
slog.Info("ctcloud started")
|
||||
err = <-monitorDone
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
slog.Error("ct monitor stopped", "error", err)
|
||||
}
|
||||
|
||||
// Graceful shutdown: stop intake, drain in-flight work, flush, then exit.
|
||||
slog.Info("shutting down")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
|
||||
defer cancel()
|
||||
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
p.Close()
|
||||
if prober != nil {
|
||||
prober.Wait()
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-shutdownCtx.Done():
|
||||
slog.Warn("shutdown timed out before pipeline drained")
|
||||
}
|
||||
|
||||
if err := httpSrv.Shutdown(shutdownCtx); err != nil {
|
||||
slog.Warn("http shutdown", "error", err)
|
||||
}
|
||||
slog.Info("stopped", "certificates", m.CTCertificates.Load(), "candidates", m.Candidates.Load(), "rows", m.DBRows.Load())
|
||||
return nil
|
||||
}
|
||||
|
||||
func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ready *atomic.Bool) *http.Server {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", m.Handler())
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok\n"))
|
||||
})
|
||||
mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if !ready.Load() {
|
||||
http.Error(w, "ct monitor not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if err := st.Ping(ctx); err != nil {
|
||||
http.Error(w, "database unreachable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ready\n"))
|
||||
})
|
||||
srv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
go func() {
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
slog.Error("metrics server", "error", err)
|
||||
}
|
||||
}()
|
||||
return srv
|
||||
}
|
||||
|
||||
func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
slog.Info("stats",
|
||||
"certificates", m.CTCertificates.Load(),
|
||||
"identifiers", m.CTIdentifiers.Load(),
|
||||
"direct_matches", m.DirectMatches.Load(),
|
||||
"cname_matches", m.CNAMEMatches.Load(),
|
||||
"candidates", m.Candidates.Load(),
|
||||
"dns_queries", m.DNSQueries.Load(),
|
||||
"dns_cache_hits", m.DNSCacheHits.Load(),
|
||||
"dns_errors", m.DNSErrors.Load(),
|
||||
"db_rows", m.DBRows.Load(),
|
||||
"db_errors", m.DBErrors.Load(),
|
||||
"queue_drops", m.QueueDrops.Load(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user