bucketeer/cmd/ctcloud/main.go

// Command ctcloud continuously consumes Certificate Transparency logs through
// embedded CertSpotter internals, classifies cloud/database/storage resources
// in-process and persists only useful, enriched findings to PostgreSQL.
package main

import (
	"context"
	"errors"
	"log/slog"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync/atomic"
	"syscall"
	"time"

	"software.sslmate.com/src/certspotter/monitor"

	"tea.karlcom.de/karl/bucketeer/internal/config"
	"tea.karlcom.de/karl/bucketeer/internal/dns"
	"tea.karlcom.de/karl/bucketeer/internal/metrics"
	"tea.karlcom.de/karl/bucketeer/internal/model"
	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
	"tea.karlcom.de/karl/bucketeer/internal/probe"
	"tea.karlcom.de/karl/bucketeer/internal/store"
)

func main() {
	if err := run(); err != nil {
		slog.Error("fatal", "error", err)
		os.Exit(1)
	}
}

func levelFromString(s string) slog.Level {
	switch s {
	case "debug":
		return slog.LevelDebug
	case "warn":
		return slog.LevelWarn
	case "error":
		return slog.LevelError
	default:
		return slog.LevelInfo
	}
}

func run() error {
	cfg, err := config.Load()
	if err != nil {
		return err
	}
	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: levelFromString(cfg.LogLevel)})))
	if strings.Contains(cfg.DatabaseURL, "sslmode=disable") {
		slog.Warn("DATABASE_URL has sslmode=disable; only use this on a private network")
	}

	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()

	m := metrics.New()

	st, err := store.Open(ctx, cfg.DatabaseURL, cfg.DBMaxConns)
	if err != nil {
		return err
	}
	defer st.Close()
	st.SetMetrics(m)
	if err := st.Migrate(ctx); err != nil {
		return err
	}
	slog.Info("database ready")

	enricher := &dns.Enricher{
		Resolver:    dns.NewServerPool(cfg.DNSServers),
		Cache:       dns.NewTTLCache[dns.Result](cfg.DNSCacheSize, 64),
		PositiveTTL: cfg.DNSPositiveTTL,
		NegativeTTL: cfg.DNSNegativeTTL,
		Timeout:     cfg.DNSTimeout,
		MaxDepth:    cfg.DNSMaxCNAMEdepth,
		Metrics:     m,
	}

	var prober *probe.Prober
	if cfg.ActiveProbing {
		prober = probe.New(probe.Config{
			Workers: cfg.ProbeWorkers,
			Timeout: cfg.ProbeTimeout,
			Metrics: m,
			Update: func(uctx context.Context, c model.Candidate, state model.ExposureState) {
				_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state)
			},
		})
		prober.Start(ctx)
		slog.Info("active probing enabled")
	}

	write := func(wctx context.Context, batch []pipeline.Finding) error {
		err := st.WriteFindings(wctx, batch)
		if err == nil && prober != nil {
			for i := range batch {
				prober.Submit(batch[i].Candidate)
			}
		}
		return err
	}

	p := pipeline.New(pipeline.Config{
		DirectQueueSize: cfg.DirectQueueSize,
		DNSQueueSize:    cfg.DNSQueueSize,
		DBQueueSize:     cfg.DBQueueSize,
		DNSWorkers:      cfg.DNSWorkers,
		BatchSize:       cfg.DBBatchSize,
		FlushInterval:   cfg.DBFlushInterval,
		DedupTTL:        cfg.DedupeTTL,
		DedupSize:       cfg.DedupSize,
		Enricher:        enricher,
		Write:           write,
		Metrics:         m,
	})
	p.Start(ctx)

	var monitorReady atomic.Bool
	monitorCfg := &monitor.Config{
		LogListSource:       cfg.LogListSource,
		State:               st,
		StartAtEnd:          cfg.StartAtEnd,
		Verbose:             cfg.LogLevel == "debug",
		HealthCheckInterval: 24 * time.Hour,
		IdentifierSink:      p,
	}

	monitorDone := make(chan error, 1)
	go func() {
		monitorReady.Store(true)
		monitorDone <- monitor.Run(ctx, monitorCfg)
	}()

	httpSrv := startHTTP(cfg.MetricsAddr, m, st, &monitorReady)
	go logStats(ctx, m, cfg.StatsInterval)

	slog.Info("ctcloud started")
	err = <-monitorDone
	if err != nil && !errors.Is(err, context.Canceled) {
		slog.Error("ct monitor stopped", "error", err)
	}

	// Graceful shutdown: stop intake, drain in-flight work, flush, then exit.
	slog.Info("shutting down")
	shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
	defer cancel()

	drained := make(chan struct{})
	go func() {
		p.Close()
		if prober != nil {
			prober.Wait()
		}
		close(drained)
	}()
	select {
	case <-drained:
	case <-shutdownCtx.Done():
		slog.Warn("shutdown timed out before pipeline drained")
	}

	if err := httpSrv.Shutdown(shutdownCtx); err != nil {
		slog.Warn("http shutdown", "error", err)
	}
	slog.Info("stopped", "certificates", m.CTCertificates.Load(), "candidates", m.Candidates.Load(), "rows", m.DBRows.Load())
	return nil
}

func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ready *atomic.Bool) *http.Server {
	mux := http.NewServeMux()
	mux.Handle("/metrics", m.Handler())
	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("ok\n"))
	})
	mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
		ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
		defer cancel()
		if !ready.Load() {
			http.Error(w, "ct monitor not initialized", http.StatusServiceUnavailable)
			return
		}
		if err := st.Ping(ctx); err != nil {
			http.Error(w, "database unreachable", http.StatusServiceUnavailable)
			return
		}
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("ready\n"))
	})
	srv := &http.Server{
		Addr:              addr,
		Handler:           mux,
		ReadHeaderTimeout: 5 * time.Second,
		ReadTimeout:       10 * time.Second,
		WriteTimeout:      10 * time.Second,
		IdleTimeout:       60 * time.Second,
	}
	go func() {
		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
			slog.Error("metrics server", "error", err)
		}
	}()
	return srv
}

func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			slog.Info("stats",
				"certificates", m.CTCertificates.Load(),
				"identifiers", m.CTIdentifiers.Load(),
				"direct_matches", m.DirectMatches.Load(),
				"cname_matches", m.CNAMEMatches.Load(),
				"candidates", m.Candidates.Load(),
				"dns_queries", m.DNSQueries.Load(),
				"dns_cache_hits", m.DNSCacheHits.Load(),
				"dns_errors", m.DNSErrors.Load(),
				"db_rows", m.DBRows.Load(),
				"db_errors", m.DBErrors.Load(),
				"queue_drops", m.QueueDrops.Load(),
			)
		}
	}
}