ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

internal/probe/probe.go

@@ -0,0 +1,142 @@
+// Package probe performs optional, minimal, non-destructive reachability checks.
+// It never authenticates, downloads objects, mutates anything, or runs database
+// queries; it only observes the response to a single HTTPS GET /.
+package probe
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+// UpdateFunc records the observed exposure state.
+type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState)
+
+type Config struct {
+	Workers int
+	Timeout time.Duration
+	Queue   int
+	Metrics *metrics.Metrics
+	Update  UpdateFunc
+}
+
+// Prober runs a fixed pool of workers over a bounded queue.
+type Prober struct {
+	cfg    Config
+	queue  chan model.Candidate
+	client *http.Client
+	wg     sync.WaitGroup
+}
+
+func New(cfg Config) *Prober {
+	if cfg.Workers <= 0 {
+		cfg.Workers = 4
+	}
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = 5 * time.Second
+	}
+	if cfg.Queue <= 0 {
+		cfg.Queue = 1024
+	}
+	return &Prober{
+		cfg:   cfg,
+		queue: make(chan model.Candidate, cfg.Queue),
+		client: &http.Client{
+			Timeout: cfg.Timeout,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+			Transport: &http.Transport{
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true}, // observation only, no trust decision
+				MaxIdleConns:        cfg.Workers * 2,
+				MaxIdleConnsPerHost: 2,
+				IdleConnTimeout:     30 * time.Second,
+				DialContext: (&net.Dialer{
+					Timeout:   cfg.Timeout,
+					KeepAlive: 30 * time.Second,
+				}).DialContext,
+			},
+		},
+	}
+}
+
+func (p *Prober) Start(ctx context.Context) {
+	for i := 0; i < p.cfg.Workers; i++ {
+		p.wg.Add(1)
+		go p.worker(ctx)
+	}
+}
+
+// Submit enqueues a candidate without blocking; it drops when the queue is full.
+func (p *Prober) Submit(c model.Candidate) {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.ProbeAttempts.Add(1)
+	}
+	select {
+	case p.queue <- c:
+	default:
+	}
+}
+
+func (p *Prober) Wait() { p.wg.Wait() }
+
+func (p *Prober) worker(ctx context.Context) {
+	defer p.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case c, ok := <-p.queue:
+			if !ok {
+				return
+			}
+			state := p.probe(ctx, c)
+			if p.cfg.Metrics != nil && state == model.ExposurePublicRead {
+				p.cfg.Metrics.ProbePublic.Add(1)
+			}
+			if p.cfg.Update != nil {
+				p.cfg.Update(ctx, c, state)
+			}
+		}
+	}
+}
+
+func (p *Prober) probe(ctx context.Context, c model.Candidate) model.ExposureState {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+c.Hostname+"/", nil)
+	if err != nil {
+		return model.ExposureUnknown
+	}
+	resp, err := p.client.Do(req)
+	if err != nil {
+		var dnsErr *net.DNSError
+		if errors.As(err, &dnsErr) {
+			return model.ExposurePrivate
+		}
+		return model.ExposureUnknown
+	}
+	defer resp.Body.Close()
+
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		return model.ExposurePublicRead
+	case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized:
+		return model.ExposureAuthRequired
+	case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone:
+		return model.ExposureNotFound
+	default:
+		return model.ExposureReachable
+	}
+}
+
+// Describe is a helper for logs/tests.
+func Describe(s model.ExposureState) string {
+	return fmt.Sprintf("exposure=%s", s)
+}

internal/probe/probe_test.go

@@ -0,0 +1,45 @@
+package probe
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func newProber(t *testing.T, status int) (*Prober, string) {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(status)
+	}))
+	t.Cleanup(srv.Close)
+	p := New(Config{Workers: 1, Timeout: time.Second})
+	// httptest uses a self-signed cert; the prober already skips verification.
+	_ = tls.Config{}
+	host := strings.TrimPrefix(srv.URL, "https://")
+	return p, host
+}
+
+func TestProbeClassification(t *testing.T) {
+	cases := []struct {
+		status int
+		want   model.ExposureState
+	}{
+		{http.StatusOK, model.ExposurePublicRead},
+		{http.StatusForbidden, model.ExposureAuthRequired},
+		{http.StatusUnauthorized, model.ExposureAuthRequired},
+		{http.StatusNotFound, model.ExposureNotFound},
+		{http.StatusInternalServerError, model.ExposureReachable},
+	}
+	for _, c := range cases {
+		p, host := newProber(t, c.status)
+		got := p.probe(t.Context(), model.Candidate{Hostname: host})
+		if got != c.want {
+			t.Errorf("status %d: got %s, want %s", c.status, got, c.want)
+		}
+	}
+}