ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
This commit is contained in:
StasonJatham committed
1 parent
9e78a114ac
commit
e8f7d6bfdf
118 files changed
+16473
No files matched your search
@@ -0,0 +1,160 @@
|
||||
// Package config loads all ctcloud configuration from environment variables.
|
||||
// There are no configuration files at runtime.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int
|
||||
|
||||
DNSServers []string
|
||||
DNSWorkers int
|
||||
DNSTimeout time.Duration
|
||||
DNSCacheSize int
|
||||
DNSMaxCNAMEdepth int
|
||||
DNSPositiveTTL time.Duration
|
||||
DNSNegativeTTL time.Duration
|
||||
|
||||
DirectQueueSize int
|
||||
DNSQueueSize int
|
||||
DBQueueSize int
|
||||
|
||||
DBBatchSize int
|
||||
DBFlushInterval time.Duration
|
||||
|
||||
ActiveProbing bool
|
||||
ProbeWorkers int
|
||||
ProbeTimeout time.Duration
|
||||
|
||||
MetricsAddr string
|
||||
LogLevel string
|
||||
StatsInterval time.Duration
|
||||
|
||||
LogListSource string
|
||||
StartAtEnd bool
|
||||
ShutdownTimeout time.Duration
|
||||
DedupeTTL time.Duration
|
||||
}
|
||||
|
||||
// DefaultLogListSource is the public CT log list used by certspotter.
|
||||
const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.json"
|
||||
|
||||
// DefaultDNSServers are load-balanced across. Set DNS_SERVERS=system to use the
|
||||
// host resolver instead, or provide your own comma-separated list.
|
||||
var DefaultDNSServers = []string{"1.1.1.1:53", "8.8.8.8:53", "1.0.0.1:53"}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
c := &Config{
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
DBMaxConns: envInt("DB_MAX_CONNS", 8),
|
||||
DNSServers: envList("DNS_SERVERS", DefaultDNSServers),
|
||||
DNSWorkers: envInt("DNS_WORKERS", 16),
|
||||
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
|
||||
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<20),
|
||||
DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
|
||||
DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
|
||||
DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
|
||||
DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192),
|
||||
DNSQueueSize: envInt("DNS_QUEUE_SIZE", 65536),
|
||||
DBQueueSize: envInt("DB_QUEUE_SIZE", 8192),
|
||||
DBBatchSize: envInt("DB_BATCH_SIZE", 500),
|
||||
DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
|
||||
ActiveProbing: envBool("ACTIVE_PROBING", false),
|
||||
ProbeWorkers: envInt("PROBE_WORKERS", 4),
|
||||
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
|
||||
MetricsAddr: envString("METRICS_ADDR", ":9090"),
|
||||
LogLevel: envString("LOG_LEVEL", "info"),
|
||||
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
|
||||
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
|
||||
StartAtEnd: envBool("START_AT_END", false),
|
||||
ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
|
||||
DedupeTTL: envDuration("DEDUP_TTL", time.Hour),
|
||||
}
|
||||
if err := c.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Config) validate() error {
|
||||
if c.DatabaseURL == "" {
|
||||
return fmt.Errorf("DATABASE_URL is required")
|
||||
}
|
||||
if c.DNSWorkers < 1 {
|
||||
return fmt.Errorf("DNS_WORKERS must be >= 1")
|
||||
}
|
||||
if c.DNSMaxCNAMEdepth < 1 {
|
||||
return fmt.Errorf("DNS_MAX_CNAME_DEPTH must be >= 1")
|
||||
}
|
||||
if c.DirectQueueSize < 1 || c.DNSQueueSize < 1 || c.DBQueueSize < 1 {
|
||||
return fmt.Errorf("queue sizes must be >= 1")
|
||||
}
|
||||
if c.DBBatchSize < 1 {
|
||||
return fmt.Errorf("DB_BATCH_SIZE must be >= 1")
|
||||
}
|
||||
switch c.LogLevel {
|
||||
case "debug", "info", "warn", "error":
|
||||
default:
|
||||
return fmt.Errorf("LOG_LEVEL must be one of debug, info, warn, error")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func envList(key string, def []string) []string {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
parts := strings.Split(v, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return def
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func envString(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func envBool(key string, def bool) bool {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
|
||||
return b
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func envDuration(key string, def time.Duration) time.Duration {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if d, err := time.ParseDuration(strings.TrimSpace(v)); err == nil {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadDefaults(t *testing.T) {
|
||||
t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.DatabaseURL != "postgres://localhost/ctcloud" {
|
||||
t.Fatalf("database url = %q", c.DatabaseURL)
|
||||
}
|
||||
if c.DNSWorkers != 16 || c.DNSMaxCNAMEdepth != 8 {
|
||||
t.Fatalf("unexpected DNS defaults: %+v", c)
|
||||
}
|
||||
if c.ActiveProbing {
|
||||
t.Fatal("active probing must default to false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOverrides(t *testing.T) {
|
||||
t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
|
||||
t.Setenv("DNS_WORKERS", "4")
|
||||
t.Setenv("ACTIVE_PROBING", "true")
|
||||
t.Setenv("DB_FLUSH_INTERVAL", "250ms")
|
||||
t.Setenv("STATS_INTERVAL", "30s")
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.DNSWorkers != 4 || !c.ActiveProbing {
|
||||
t.Fatalf("overrides not applied: %+v", c)
|
||||
}
|
||||
if c.DBFlushInterval != 250*time.Millisecond || c.StatsInterval != 30*time.Second {
|
||||
t.Fatalf("duration overrides not applied: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalid(t *testing.T) {
|
||||
t.Setenv("DATABASE_URL", "")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("expected error without DATABASE_URL")
|
||||
}
|
||||
t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
|
||||
t.Setenv("LOG_LEVEL", "verbose")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("expected error for bad LOG_LEVEL")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user