ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.gitignore
+pgdata
+*.test
+*.out
+.gocache

.env.example

@@ -0,0 +1,16 @@
+# Copy to .env and edit. .env is gitignored.
+
+POSTGRES_USER=ctcloud
+POSTGRES_PASSWORD=change-me-strong
+POSTGRES_DB=ctcloud
+
+# Local compose (private bridge network, no TLS):
+DATABASE_URL=postgres://ctcloud:change-me-strong@postgres:5432/ctcloud?sslmode=disable
+
+# Production: use a TLS-enabled database instead, e.g.
+# DATABASE_URL=postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full
+
+DNS_SERVERS=1.1.1.1:53,8.8.8.8:53,1.0.0.1:53
+DNS_WORKERS=16
+LOG_LEVEL=info
+ACTIVE_PROBING=false

.gitignore

@@ -0,0 +1,5 @@
+/ctcloud
+*.test
+*.out
+pgdata/
+.env

ARCHITECTURE.md

@@ -0,0 +1,92 @@
+# Architecture
+
+## Data flow
+
+```text
+Certificate Transparency Logs
+             │
+             ▼
+     CertSpotter internals            fetch / verify / parse (third_party)
+   (in-process, no stdout/JSON/files)
+             │
+             ▼
+       DNS identifiers
+             │
+             ▼
+        FAST CLASSIFIER              0 allocs/op, no regexp/JSON/DNS/DB
+        ┌────┴─────┐
+   direct cloud   normal
+    resource      hostname
+        │            │
+        │            ▼
+        │      DNS/CNAME enrichment   bounded workers + TTL cache
+        │            │
+        │            ▼
+        │      classify CNAME targets
+        │            │
+        └─────┬──────┘
+              ▼
+        dedup/enrichment             TTL caches; DB uniqueness is final
+              │
+              ▼
+    optional safe reachability       HTTPS GET / only, OFF by default
+              ▼
+        batch DB writer              bounded queue, pgx batches
+              ▼
+          PostgreSQL
+```
+
+## Stages and what is allowed
+
+| Stage | May do | Must not do |
+| --- | --- | --- |
+| CertSpotter fetch/verify/parse | network, crypto, in-memory DER | persist certs, write files |
+| Identifier callback | iterate `DNSNames`, call classifier | block, log per certificate, allocate needlessly |
+| Classifier | byte/string comparisons, label scan | regexp, JSON, reflection, DNS, HTTP, DB, files, per-id logging |
+| Direct queue | bound, non-blocking send | unbounded growth |
+| DNS enrichment | DNS via worker pool, TTL cache, CNAME follow | unbounded goroutines, persistence |
+| Dedup | memory-bounded TTL caches | unbounded maps |
+| Probe (optional) | DNS/TCP/TLS/HTTP HEAD/GET `/` | auth, downloads, mutation, DB or shell access |
+| DB writer | batched `pgx.Batch` writes | one synchronous row per event |
+| Checkpoints | JSON CT position in `ct_checkpoints`/`ct_sths` | certificate/issuer contents in DB |
+
+## Concurrency and backpressure
+
+* CertSpotter runs one monitor goroutine per CT log; it calls the identifier
+  sink concurrently.
+* The sink never blocks: it classifies inline and does a non-blocking send to
+  either the direct queue or the DNS queue. Full queues increment
+  `queue_drops_total` / `queue_full_total`.
+* Direct matches have a dedicated queue and worker pool, so speculative CNAME
+  enrichment can be shed first. DNS workers also skip work when the DB queue is
+  more than half full.
+* The DB writer batches by size or `DB_FLUSH_INTERVAL` and flushes on
+  shutdown.
+
+## Failure handling
+
+* DNS and CT network errors are retried by CertSpotter with bounded exponential
+  backoff; ctcloud counts them (`ct_errors_total`) without logging per event.
+* PostgreSQL writes retry a few times with backoff; persistent failure drops
+  items from the bounded DB queue rather than growing memory.
+* The process uses `context` throughout and on `SIGTERM` stops intake, drains
+  the pipeline, flushes the DB batch and exits within `SHUTDOWN_TIMEOUT`.
+
+## Metrics and health
+
+`GET /metrics` exposes counters (`ct_certificates_total`,
+`classifier_direct_matches_total`, `classifier_cname_matches_total`,
+`dns_*`, `db_*`, `queue_*`) and gauges (`queue_depth{queue="…"}`,
+`go_goroutines`, `process_system_bytes`). No hostnames or other high-cardinality
+labels are ever used.
+
+`GET /healthz` is a liveness check. `GET /readyz` verifies the CT monitor is
+initialised and PostgreSQL is reachable.
+
+## Why no files, pipes or subprocesses
+
+Everything between certificate parsing and classification stays in memory. The
+CertSpotter patch hands identifiers directly to Go code, removing serialization
+(JSON), IPC (stdout/pipe), process boundaries (subprocess/hooks) and filesystem
+I/O (certificate/state files). CT resume state is the only thing persisted, and
+it lives in PostgreSQL.

Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1
+
+FROM golang:1.25-alpine AS build
+WORKDIR /src
+
+# Cache module downloads (including the local certspotter fork's deps).
+COPY go.mod go.sum ./
+COPY third_party/certspotter/go.mod third_party/certspotter/go.sum ./third_party/certspotter/
+RUN go mod download
+
+COPY . .
+
+# Static binary: no cgo, no shell, no writable filesystem required.
+RUN CGO_ENABLED=0 GOOS=linux go build \
+        -trimpath -ldflags="-s -w" \
+        -o /out/ctcloud ./cmd/ctcloud
+
+FROM gcr.io/distroless/static-debian12:nonroot
+COPY --from=build /out/ctcloud /ctcloud
+USER nonroot:nonroot
+EXPOSE 9090
+ENTRYPOINT ["/ctcloud"]

README.md

@@ -0,0 +1,253 @@
+# ctcloud
+
+`ctcloud` continuously consumes Certificate Transparency (CT) logs, finds
+cloud storage/database/data-service resources referenced by certificates,
+enriches candidates through DNS/CNAME resolution, and persists only useful
+findings to PostgreSQL.
+
+It is a **single stateless Go process plus PostgreSQL**. CertSpotter's real CT
+fetching, validation and parsing code is embedded and called **in-process**;
+there is no stdout handoff, no JSON handoff, no hooks, no subprocesses and no
+certificate files.
+
+> A detected provider endpoint **does not** mean the underlying data is publicly
+> accessible. `ctcloud` reports facts (hostname existed in CT, resolved, answered
+> a request) separately from conclusions (`exposure_state`). Most findings are
+> simply an endpoint that exists.
+
+## How it works
+
+```text
+CT logs ──► CertSpotter internals (fetch/verify/parse)
+              │  direct Go call: IdentifierSink.HandleIdentifiers
+              ▼
+        fast classifier  (0 allocs/op, no regexp/JSON/DNS)
+        ┌──────┴───────┐
+   direct match    non-match
+        │              ▼
+        │        DNS/CNAME enrichment
+        │           (followed + classified)
+        └──────┬───────┘
+               ▼
+        dedup + bounded queues
+               ▼
+     optional safe reachability
+               ▼
+        batched PostgreSQL writer
+               ▼
+           PostgreSQL
+```
+
+See [ARCHITECTURE.md](ARCHITECTURE.md) for per-stage rules.
+
+## Why CertSpotter is embedded (and patched)
+
+Re-implementing CT log handling is a large, security-sensitive job. Upstream
+CertSpotter already does log-list handling, STH fetching/verification, Merkle
+tree maintenance, entry parsing and certificate reconstruction.
+
+The only change is a two-hunk patch in `third_party/certspotter` that adds an
+`IdentifierSink` callback. After a certificate is parsed, the identifiers are
+handed to ctcloud with a direct Go function call and the watchlist/chain/
+notification machinery is skipped:
+
+```go
+if config.IdentifierSink != nil {
+    config.IdentifierSink.HandleIdentifiers(ctx, identifiers)
+    return nil
+}
+```
+
+See [`third_party/certspotter/CTCLOUD_PATCH.md`](third_party/certspotter/CTCLOUD_PATCH.md).
+This deliberately avoids stdout piping, JSON files, hook scripts, subprocesses
+and temporary certificate files, which are all slow, serializing and
+filesystem-bound. Upstream licensing/notices are preserved and the patch is
+isolated so it can be rebased.
+
+## Supported provider patterns
+
+**AWS** — S3 (`<bucket>.s3[.dualstack].<region>.amazonaws.com`,
+`s3-website[-<region>]`), S3 access points (`s3-accesspoint[-fips]`,
+`dualstack`), multi-region access points (`mrap.accesspoint.s3-global`), S3
+Express (`s3express-*`), RDS/Aurora/proxy (`*.rds.amazonaws.com`), DocumentDB,
+Redshift (provisioned + serverless), OpenSearch (`search-*` public / `vpc-*`
+VPC), Neptune, MemoryDB, ElastiCache. Generic API endpoints
+(`s3.eu-central-1.amazonaws.com`, `dynamodb.*`, `redshift.<region>`,
+`rds.<region>`, …) are rejected.
+
+**Azure** — `blob|dfs|file|queue|table|web.core.windows.net`,
+`*.database.windows.net`, `*.postgres|mysql.database.azure.com`,
+`*.documents.azure.com`, `*.{mongo,cassandra,gremlin}.cosmos.azure.com`,
+`*.redis.cache.windows.net`, `*.redis.azure.net`, `*.search.windows.net`.
+Bare service endpoints are rejected.
+
+**Google Cloud / Firebase** — `<bucket>.storage.googleapis.com`,
+`<db>.firebaseio.com`, `<db>.<region>.firebasedatabase.app`,
+`<project>.appspot.com`. `storage.googleapis.com` itself is rejected.
+
+**Cloudflare R2** — `<account>.r2.cloudflarestorage.com`; `<account>.r2.dev`
+is flagged as an explicitly public URL form.
+
+**DigitalOcean** — `<bucket>.<region>[.cdn].digitaloceanspaces.com`,
+`*.db.ondigitalocean.com`. `<region>.digitaloceanspaces.com` is rejected.
+
+**Supabase** — `<ref>.supabase.co`, `db.<ref>.supabase.co` (PostgreSQL),
+`aws-<n>-<region>.pooler.supabase.com`.
+
+**Oracle Cloud (OCI)** — `<namespace>.objectstorage|swiftobjectstorage.<region>.oci.customer-oci.com`
+(resource-specific), and the generic `objectstorage.<region>.oraclecloud.com`
+forms (`resource_specific = false`; not stored as findings).
+
+Provider-specific details live in `internal/classify/`.
+
+## Configuration
+
+Environment variables only; no config files, no runtime pattern files.
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `DATABASE_URL` | *(required)* | PostgreSQL connection string |
+| `DB_MAX_CONNS` | `8` | pgx pool size |
+| `DNS_SERVERS` | `1.1.1.1:53,8.8.8.8:53,1.0.0.1:53` | comma-separated resolvers (round-robin + failover); `system` for the host resolver |
+| `DNS_WORKERS` | `16` | DNS enrichment workers |
+| `DNS_TIMEOUT` | `5s` | per-lookup timeout |
+| `DNS_CACHE_SIZE` | `1048576` | max DNS cache entries |
+| `DNS_MAX_CNAME_DEPTH` | `8` | max CNAME hops |
+| `DNS_POSITIVE_TTL` / `DNS_NEGATIVE_TTL` | `10m` / `1m` | DNS cache TTLs |
+| `DIRECT_QUEUE_SIZE` | `8192` | direct-match queue capacity |
+| `DNS_QUEUE_SIZE` | `65536` | DNS-enrichment queue capacity |
+| `DB_QUEUE_SIZE` | `8192` | DB-write queue capacity |
+| `DB_BATCH_SIZE` | `500` | findings per write batch |
+| `DB_FLUSH_INTERVAL` | `2s` | max time before a batch is flushed |
+| `ACTIVE_PROBING` | `false` | enable optional reachability checks |
+| `PROBE_WORKERS` / `PROBE_TIMEOUT` | `4` / `5s` | probe pool |
+| `DEDUP_TTL` | `1h` | hostname/finding dedup window |
+| `METRICS_ADDR` | `:9090` | metrics/health listener |
+| `LOG_LEVEL` | `info` | `debug\|info\|warn\|error` |
+| `STATS_INTERVAL` | `60s` | aggregated stats log interval |
+| `LOG_LIST_SOURCE` | Google log list v3 | certspotter log list URL |
+| `START_AT_END` | `false` | skip existing backlog on first run |
+| `SHUTDOWN_TIMEOUT` | `20s` | graceful shutdown budget |
+
+## Running with Docker
+
+```sh
+cp .env.example .env      # set a strong POSTGRES_PASSWORD and matching DATABASE_URL
+docker compose up --build
+```
+
+That starts `postgres` and the stateless `ctcloud` container. The application
+container runs as non-root, has a read-only root filesystem, drops all Linux
+capabilities, sets CPU/memory/pids limits and mounts no application volume. The
+only persistent state is PostgreSQL (CT checkpoints + cloud assets).
+
+Endpoints: `GET /metrics`, `GET /healthz`, `GET /readyz` on port `9090`, bound
+to `127.0.0.1` only. The listener has no authentication, so scrape it locally or
+from a sidecar and never expose it publicly.
+
+## Production notes
+
+* **Secrets** — database credentials live in `.env` (gitignored); `.env.example`
+  is the template. Inject the same variables from your orchestrator's secret
+  store in production.
+* **Database TLS** — the local compose stack uses `sslmode=disable` because the
+  compose bridge network is private. For production, point `DATABASE_URL` at a
+  TLS-enabled database, e.g.
+  `postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full`.
+  pgx handles TLS natively; no extra flags are needed.
+* **Metrics** — bound to loopback. There is no auth, no debug endpoints and no
+  high-cardinality labels (never hostnames).
+* **Resource limits** — `mem_limit`/`cpus`/`pids_limit` are set in compose so
+  overload is bounded by the queue sizes rather than by the OOM killer.
+* **Authorization** — leave `ACTIVE_PROBING=false` unless you are authorised to
+  probe the discovered infrastructure.
+
+## Database schema
+
+* `cloud_assets` — one row per `(provider, service, provider_hostname)`, with
+  `hostname` (original CT hostname), `resource_identifier`, `region`,
+  `resource_specific`, `public_semantic`, `exposure_state`, `first_seen`,
+  `last_seen`.
+* `observations` — append-only observations linking an asset to a
+  `source_hostname`, `source_type` (`direct_ct`/`cname`), CNAME chain and
+  A/AAAA records.
+* `ct_checkpoints` — per-log CT resume state (JSON-encoded certspotter
+  `LogState`; Merkle positions, no certificates).
+* `ct_sths` — downloaded, unverified signed tree heads needed for safe resume.
+
+Schema is applied idempotently at startup from embedded SQL in `migrations/`.
+No certificate contents are ever stored.
+
+## Performance characteristics
+
+Optimised in this order: I/O off the hot path, then serialization, then process
+boundaries, then allocations, then cheap suffix/label matching, then bounded
+pools, then DNS caching, dedup and batched writes.
+
+The classifier:
+
+* no regexp, JSON, reflection, DNS, HTTP, filesystem, DB or logging;
+* no `strings.Split`, no `fmt.Sprintf`;
+* lowercases only when an uppercase byte is present;
+* parses labels by byte offsets, returning substrings (zero copy);
+* fixed stack arrays, no unbounded maps or goroutines.
+
+### Benchmark results (Apple M4, Go 1.27)
+
+```text
+BenchmarkClassifyAWS-10            60.03 ns/op    0 B/op   0 allocs/op
+BenchmarkClassifyAzure-10          44.93 ns/op    0 B/op   0 allocs/op
+BenchmarkRejectNormalDomain-10     30.28 ns/op    0 B/op   0 allocs/op
+BenchmarkClassifyMixed-10         369.50 ns/op    0 B/op   0 allocs/op
+```
+
+The whole in-process sink path is allocation-free too:
+
+```text
+BenchmarkHandleHostNonMatch-10      115.6 ns/op    0 B/op   0 allocs/op
+BenchmarkHandleHostDirectMatch-10  155.7 ns/op    0 B/op   0 allocs/op
+```
+
+Run them yourself:
+
+```sh
+go test ./internal/classify/ ./internal/pipeline/ -run '^$' -bench . -benchmem
+```
+
+`BenchmarkClassifyMixed` and `TestHandleHostNoAllocs` fail unless the result is
+exactly 0 allocs/op, so the target is enforced by the test suite.
+
+### DNS load balancing
+
+DNS lookups are round-robined across `DNS_SERVERS`. Each attempt is bounded
+(2s) and fails over to the next server on timeout/refused errors, so one dead
+resolver does not stall lookups. Definitive answers (NXDOMAIN) are returned
+immediately rather than retried on every server, so negative lookups do not
+multiply DNS traffic.
+
+## Testing
+
+```sh
+go test ./...          # unit + table-driven provider tests
+go test -race ./...    # race detector
+go vet ./...
+```
+
+Integration tests for PostgreSQL run only when `DATABASE_URL` is set:
+
+```sh
+DATABASE_URL=postgres://ctcloud:ctcloud@localhost:5432/ctcloud go test ./internal/store/
+```
+
+## Limitations
+
+* `net.Resolver.LookupCNAME` returns the final canonical name, so intermediate
+  CNAME hops are not observed (the final target is still classified). Switch to
+  `miekg/dns` if the full chain is required.
+* Optional reachability checks are off by default; when enabled they perform a
+  single non-destructive HTTPS `GET /` only — no authentication, downloads,
+  mutation or database queries.
+* OCI generic `oraclecloud.com` endpoints are recognised but not stored, since
+  the bucket name lives in the URL path rather than the hostname.
+* Custom R2 (and other) domains are only discovered through DNS/CNAME
+  enrichment, and only if the CT hostname itself appears in a certificate.

cmd/ctcloud/main.go

@@ -0,0 +1,234 @@
+// Command ctcloud continuously consumes Certificate Transparency logs through
+// embedded CertSpotter internals, classifies cloud/database/storage resources
+// in-process and persists only useful, enriched findings to PostgreSQL.
+package main
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"software.sslmate.com/src/certspotter/monitor"
+
+	"tea.karlcom.de/karl/bucketeer/internal/config"
+	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
+	"tea.karlcom.de/karl/bucketeer/internal/probe"
+	"tea.karlcom.de/karl/bucketeer/internal/store"
+)
+
+func main() {
+	if err := run(); err != nil {
+		slog.Error("fatal", "error", err)
+		os.Exit(1)
+	}
+}
+
+func levelFromString(s string) slog.Level {
+	switch s {
+	case "debug":
+		return slog.LevelDebug
+	case "warn":
+		return slog.LevelWarn
+	case "error":
+		return slog.LevelError
+	default:
+		return slog.LevelInfo
+	}
+}
+
+func run() error {
+	cfg, err := config.Load()
+	if err != nil {
+		return err
+	}
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: levelFromString(cfg.LogLevel)})))
+	if strings.Contains(cfg.DatabaseURL, "sslmode=disable") {
+		slog.Warn("DATABASE_URL has sslmode=disable; only use this on a private network")
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	m := metrics.New()
+
+	st, err := store.Open(ctx, cfg.DatabaseURL, cfg.DBMaxConns)
+	if err != nil {
+		return err
+	}
+	defer st.Close()
+	st.SetMetrics(m)
+	if err := st.Migrate(ctx); err != nil {
+		return err
+	}
+	slog.Info("database ready")
+
+	enricher := &dns.Enricher{
+		Resolver:    dns.NewServerPool(cfg.DNSServers),
+		Cache:       dns.NewTTLCache[dns.Result](cfg.DNSCacheSize, 64),
+		PositiveTTL: cfg.DNSPositiveTTL,
+		NegativeTTL: cfg.DNSNegativeTTL,
+		Timeout:     cfg.DNSTimeout,
+		MaxDepth:    cfg.DNSMaxCNAMEdepth,
+		Metrics:     m,
+	}
+
+	var prober *probe.Prober
+	if cfg.ActiveProbing {
+		prober = probe.New(probe.Config{
+			Workers: cfg.ProbeWorkers,
+			Timeout: cfg.ProbeTimeout,
+			Metrics: m,
+			Update: func(uctx context.Context, c model.Candidate, state model.ExposureState) {
+				_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state)
+			},
+		})
+		prober.Start(ctx)
+		slog.Info("active probing enabled")
+	}
+
+	write := func(wctx context.Context, batch []pipeline.Finding) error {
+		err := st.WriteFindings(wctx, batch)
+		if err == nil && prober != nil {
+			for i := range batch {
+				prober.Submit(batch[i].Candidate)
+			}
+		}
+		return err
+	}
+
+	p := pipeline.New(pipeline.Config{
+		DirectQueueSize: cfg.DirectQueueSize,
+		DNSQueueSize:    cfg.DNSQueueSize,
+		DBQueueSize:     cfg.DBQueueSize,
+		DNSWorkers:      cfg.DNSWorkers,
+		BatchSize:       cfg.DBBatchSize,
+		FlushInterval:   cfg.DBFlushInterval,
+		DedupTTL:        cfg.DedupeTTL,
+		Enricher:        enricher,
+		Write:           write,
+		Metrics:         m,
+	})
+	p.Start(ctx)
+
+	var monitorReady atomic.Bool
+	monitorCfg := &monitor.Config{
+		LogListSource:       cfg.LogListSource,
+		State:               st,
+		StartAtEnd:          cfg.StartAtEnd,
+		Verbose:             cfg.LogLevel == "debug",
+		HealthCheckInterval: 24 * time.Hour,
+		IdentifierSink:      p,
+	}
+
+	monitorDone := make(chan error, 1)
+	go func() {
+		monitorReady.Store(true)
+		monitorDone <- monitor.Run(ctx, monitorCfg)
+	}()
+
+	httpSrv := startHTTP(cfg.MetricsAddr, m, st, &monitorReady)
+	go logStats(ctx, m, cfg.StatsInterval)
+
+	slog.Info("ctcloud started")
+	err = <-monitorDone
+	if err != nil && !errors.Is(err, context.Canceled) {
+		slog.Error("ct monitor stopped", "error", err)
+	}
+
+	// Graceful shutdown: stop intake, drain in-flight work, flush, then exit.
+	slog.Info("shutting down")
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
+	defer cancel()
+
+	drained := make(chan struct{})
+	go func() {
+		p.Close()
+		if prober != nil {
+			prober.Wait()
+		}
+		close(drained)
+	}()
+	select {
+	case <-drained:
+	case <-shutdownCtx.Done():
+		slog.Warn("shutdown timed out before pipeline drained")
+	}
+
+	if err := httpSrv.Shutdown(shutdownCtx); err != nil {
+		slog.Warn("http shutdown", "error", err)
+	}
+	slog.Info("stopped", "certificates", m.CTCertificates.Load(), "candidates", m.Candidates.Load(), "rows", m.DBRows.Load())
+	return nil
+}
+
+func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ready *atomic.Bool) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", m.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	})
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
+		ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
+		defer cancel()
+		if !ready.Load() {
+			http.Error(w, "ct monitor not initialized", http.StatusServiceUnavailable)
+			return
+		}
+		if err := st.Ping(ctx); err != nil {
+			http.Error(w, "database unreachable", http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ready\n"))
+	})
+	srv := &http.Server{
+		Addr:              addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			slog.Error("metrics server", "error", err)
+		}
+	}()
+	return srv
+}
+
+func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			slog.Info("stats",
+				"certificates", m.CTCertificates.Load(),
+				"identifiers", m.CTIdentifiers.Load(),
+				"direct_matches", m.DirectMatches.Load(),
+				"cname_matches", m.CNAMEMatches.Load(),
+				"candidates", m.Candidates.Load(),
+				"dns_queries", m.DNSQueries.Load(),
+				"dns_cache_hits", m.DNSCacheHits.Load(),
+				"dns_errors", m.DNSErrors.Load(),
+				"db_rows", m.DBRows.Load(),
+				"db_errors", m.DBErrors.Load(),
+				"queue_drops", m.QueueDrops.Load(),
+			)
+		}
+	}
+}

docker-compose.yml

@@ -0,0 +1,66 @@
+# Local / single-host deployment.
+#
+# Secrets come from .env (gitignored). Copy .env.example to .env and set a
+# strong POSTGRES_PASSWORD plus a matching DATABASE_URL before `docker compose
+# up`. For a production database, point DATABASE_URL at a TLS-enabled endpoint,
+# e.g. postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full
+#
+# The compose bridge network is private; sslmode=disable is only acceptable here
+# because the database is not reachable from outside the stack.
+services:
+  postgres:
+    image: postgres:17-alpine
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER:-ctcloud}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
+      POSTGRES_DB: ${POSTGRES_DB:-ctcloud}
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-ctcloud} -d ${POSTGRES_DB:-ctcloud}"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    mem_limit: 1g
+    cpus: 2.0
+    pids_limit: 256
+    logging:
+      driver: json-file
+      options: { max-size: "10m", max-file: "3" }
+    restart: unless-stopped
+
+  ctcloud:
+    build: .
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env}
+      METRICS_ADDR: ":9090"
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      DNS_SERVERS: ${DNS_SERVERS:-1.1.1.1:53,8.8.8.8:53,1.0.0.1:53}
+      DNS_WORKERS: ${DNS_WORKERS:-16}
+      ACTIVE_PROBING: ${ACTIVE_PROBING:-false}
+      # START_AT_END: "true"   # skip historical certificates on first run
+    # Metrics/health are bound to loopback only: scrape from the host or a
+    # sidecar, never expose them publicly (there is no auth on this listener).
+    ports:
+      - "127.0.0.1:9090:9090"
+    # No application volume: the container is stateless apart from PostgreSQL.
+    read_only: true
+    tmpfs:
+      - /tmp:size=16m,noexec,nosuid
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    mem_limit: 1g
+    cpus: 2.0
+    pids_limit: 512
+    logging:
+      driver: json-file
+      options: { max-size: "10m", max-file: "3" }
+    restart: unless-stopped
+
+volumes:
+  pgdata:

go.mod

@@ -0,0 +1,21 @@
+module tea.karlcom.de/karl/bucketeer
+
+go 1.25.0
+
+replace software.sslmate.com/src/certspotter => ./third_party/certspotter
+
+require (
+	github.com/jackc/pgx/v5 v5.11.0
+	software.sslmate.com/src/certspotter v0.0.0
+)
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	golang.org/x/crypto v0.52.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+)

go.sum

@@ -0,0 +1,32 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
+github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/classify/aws.go

@@ -0,0 +1,148 @@
+package classify
+
+import (
+	"strings"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match {
+	return model.Match{
+		Provider:         p,
+		Service:          s,
+		PublicSemantic:   sem,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+func classifyAWS(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" {
+		return model.Match{}, false
+	}
+	if m, ok := classifyAWSS3(s); ok {
+		return m, true
+	}
+
+	// Service-after-region forms: <resource>.<region>.<service>.amazonaws.com
+	if n >= 5 {
+		switch s.label(n - 3) {
+		case "rds":
+			return awsResourceAfterRegion(s, model.ServiceRDS)
+		case "docdb":
+			return awsResourceAfterRegion(s, model.ServiceDocumentDB)
+		case "neptune":
+			return awsResourceAfterRegion(s, model.ServiceNeptune)
+		case "redshift", "redshift-serverless":
+			return awsResourceAfterRegion(s, model.ServiceRedshift)
+		case "es":
+			first := s.label(0)
+			switch {
+			case strings.HasPrefix(first, "search-"):
+				return aws(model.ProviderAWS, model.ServiceOpenSearch,
+					model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true
+			case strings.HasPrefix(first, "vpc-"):
+				return aws(model.ProviderAWS, model.ServiceOpenSearch,
+					model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true
+			}
+		case "cache":
+			return aws(model.ProviderAWS, model.ServiceElastiCache,
+				model.PublicSemanticNone, s.label(0), s.label(n-4)), true
+		}
+	}
+
+	// Service-before-region form: <resource>.memorydb.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "memorydb" {
+		return aws(model.ProviderAWS, model.ServiceMemoryDB,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+	return model.Match{}, false
+}
+
+// awsResourceAfterRegion matches <resource>.<region>.<service>.amazonaws.com
+// and rejects the bare <service>.<region>.amazonaws.com service endpoint.
+func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) {
+	n := s.n
+	if n < 5 {
+		return model.Match{}, false
+	}
+	resource := s.label(0)
+	if resource == s.label(n-3) {
+		return model.Match{}, false
+	}
+	return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true
+}
+
+func classifyAWSS3(s *scanner) (model.Match, bool) {
+	n := s.n
+
+	// Access points: <ap>.s3-accesspoint[-fips][.dualstack].<region>.amazonaws.com
+	if n >= 5 {
+		svcIdx := -1
+		switch {
+		case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips":
+			svcIdx = n - 4
+		case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint":
+			svcIdx = n - 5
+		}
+		if svcIdx > 0 {
+			res := s.label(0)
+			if res != "s3-accesspoint" && res != "s3-accesspoint-fips" {
+				return aws(model.ProviderAWS, model.ServiceS3,
+					model.PublicSemanticNone, res, s.label(n-3)), true
+			}
+		}
+	}
+
+	// Multi-region access point: <res>.mrap.accesspoint.s3-global.amazonaws.com
+	if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" &&
+		s.label(n-3) == "s3-global" {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), ""), true
+	}
+
+	// S3 Express: <bucket>.s3express-<zone>.<region>.amazonaws.com
+	if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+
+	// S3 website: <bucket>.s3-website.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "s3-website" {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), s.label(n-3)), true
+	}
+	// S3 website (region in the service label): <bucket>.s3-website-<region>.amazonaws.com
+	if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") {
+		return aws(model.ProviderAWS, model.ServiceS3,
+			model.PublicSemanticNone, s.label(0), ""), true
+	}
+
+	// Plain S3 without region: <bucket>.s3.amazonaws.com
+	if n == 4 && s.label(n-3) == "s3" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true
+	}
+	// Plain S3 with region: <bucket>.s3.<region>.amazonaws.com
+	if n >= 5 && s.label(n-4) == "s3" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
+	}
+	// Dualstack S3: <bucket>.s3.dualstack.<region>.amazonaws.com
+	if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" {
+		res := s.label(0)
+		if res == "s3" {
+			return model.Match{}, false
+		}
+		return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true
+	}
+	return model.Match{}, false
+}

internal/classify/azure.go

@@ -0,0 +1,113 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+func azure(service model.Service, resource, region string) model.Match {
+	return model.Match{
+		Provider:         model.ProviderAzure,
+		Service:          service,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+// classifyAzureCore handles *.blob|dfs|file|queue|table|web.core.windows.net.
+func classifyAzureCore(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "core" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 4)
+	resource := s.label(0)
+	if resource == "" {
+		return model.Match{}, false
+	}
+	switch svc {
+	case "blob", "dfs":
+		return azure(model.ServiceBlobStorage, resource, ""), true
+	case "file":
+		return azure(model.ServiceFileStorage, resource, ""), true
+	case "queue":
+		return azure(model.ServiceQueue, resource, ""), true
+	case "table":
+		return azure(model.ServiceTableStorage, resource, ""), true
+	case "web":
+		return azure(model.ServiceWebApp, resource, ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureDatabase handles <server>.database.windows.net.
+func classifyAzureDatabase(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "database" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceDatabase, s.label(0), ""), true
+}
+
+// classifyAzureFlexibleServer handles <server>.{postgres,mysql}.database.azure.com.
+func classifyAzureFlexibleServer(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "database" {
+		return model.Match{}, false
+	}
+	switch s.label(n - 4) {
+	case "postgres":
+		return azure(model.ServicePostgres, s.label(0), ""), true
+	case "mysql":
+		return azure(model.ServiceMySQL, s.label(0), ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureDocuments handles <account>.documents.azure.com (Cosmos DB).
+func classifyAzureDocuments(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "documents" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceCosmosDB, s.label(0), ""), true
+}
+
+// classifyAzureCosmos handles <account>.{mongo,cassandra,gremlin}.cosmos.azure.com.
+func classifyAzureCosmos(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "cosmos" {
+		return model.Match{}, false
+	}
+	switch s.label(n - 4) {
+	case "mongo", "cassandra", "gremlin":
+		return azure(model.ServiceCosmosDB, s.label(0), ""), true
+	}
+	return model.Match{}, false
+}
+
+// classifyAzureRedis handles <name>.redis.cache.windows.net.
+func classifyAzureRedis(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" ||
+		s.label(n-3) != "cache" || s.label(n-4) != "redis" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceRedis, s.label(0), ""), true
+}
+
+// classifyAzureManagedRedis handles <name>.redis.azure.net.
+func classifyAzureManagedRedis(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "azure" || s.label(n-3) != "redis" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceRedis, s.label(0), ""), true
+}
+
+// classifyAzureSearch handles <service>.search.windows.net.
+func classifyAzureSearch(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "search" {
+		return model.Match{}, false
+	}
+	return azure(model.ServiceSearch, s.label(0), ""), true
+}

internal/classify/classify.go

@@ -0,0 +1,198 @@
+// Package classify maps a DNS hostname to a cloud/database/storage resource.
+//
+// The hot path is deliberately allocation-free: it never lowercases unless an
+// uppercase byte is present, never splits the hostname into a []string, never
+// uses regexp/JSON/reflection, and returns substrings of the input as the
+// region/resource. See BenchmarkClassify* for the 0 allocs/op assertion.
+package classify
+
+import (
+	"strings"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+const maxLabels = 64
+
+// scanner holds the byte offsets of each dot-separated label of a hostname.
+// It is stack allocated by Classify and never heap allocated.
+type scanner struct {
+	host  string
+	n     int
+	start [maxLabels]int32
+}
+
+func newScanner(host string) scanner {
+	var s scanner
+	s.host = host
+	start := int32(0)
+	for i := 0; i < len(host); i++ {
+		if host[i] == '.' {
+			if s.n >= maxLabels {
+				s.n = -1
+				return s
+			}
+			s.start[s.n] = start
+			s.n++
+			start = int32(i + 1)
+		}
+	}
+	if s.n >= maxLabels {
+		s.n = -1
+		return s
+	}
+	s.start[s.n] = start
+	s.n++
+	return s
+}
+
+// label returns the i-th label (0 = leftmost). Callers must ensure i < n.
+func (s *scanner) label(i int) string {
+	end := int32(len(s.host))
+	if i+1 < s.n {
+		end = s.start[i+1] - 1
+	}
+	return s.host[s.start[i]:end]
+}
+
+func hasUpper(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= 'A' && s[i] <= 'Z' {
+			return true
+		}
+	}
+	return false
+}
+
+func trimTrailingDots(s string) string {
+	for len(s) > 0 && s[len(s)-1] == '.' {
+		s = s[:len(s)-1]
+	}
+	return s
+}
+
+func validHost(host string) bool {
+	if host == "" || len(host) > 253 {
+		return false
+	}
+	for i := 0; i < len(host); i++ {
+		c := host[i]
+		switch {
+		case c >= 'a' && c <= 'z':
+		case c >= '0' && c <= '9':
+		case c == '-' || c == '.' || c == '_':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+// Classify returns the provider resource described by host, if any.
+// The returned Match's string fields alias host.
+func Classify(host string) (model.Match, bool) {
+	host = trimTrailingDots(host)
+	if host == "" || len(host) > 253 {
+		return model.Match{}, false
+	}
+	// Wildcard certificates never describe a concrete resource.
+	if host[0] == '*' {
+		return model.Match{}, false
+	}
+	if hasUpper(host) {
+		host = strings.ToLower(host)
+	}
+	if !validHost(host) {
+		return model.Match{}, false
+	}
+
+	s := newScanner(host)
+	if s.n < 2 {
+		return model.Match{}, false
+	}
+
+	switch {
+	case strings.HasSuffix(host, ".amazonaws.com"):
+		return classifyAWS(&s)
+	case strings.HasSuffix(host, ".core.windows.net"):
+		return classifyAzureCore(&s)
+	case strings.HasSuffix(host, ".database.windows.net"):
+		return classifyAzureDatabase(&s)
+	case strings.HasSuffix(host, ".database.azure.com"):
+		return classifyAzureFlexibleServer(&s)
+	case strings.HasSuffix(host, ".documents.azure.com"):
+		return classifyAzureDocuments(&s)
+	case strings.HasSuffix(host, ".cosmos.azure.com"):
+		return classifyAzureCosmos(&s)
+	case strings.HasSuffix(host, ".cache.windows.net"):
+		return classifyAzureRedis(&s)
+	case strings.HasSuffix(host, ".redis.azure.net"):
+		return classifyAzureManagedRedis(&s)
+	case strings.HasSuffix(host, ".search.windows.net"):
+		return classifyAzureSearch(&s)
+	case strings.HasSuffix(host, ".storage.googleapis.com"):
+		return classifyGCS(&s)
+	case strings.HasSuffix(host, ".firebaseio.com"):
+		return classifyFirebaseIO(&s)
+	case strings.HasSuffix(host, ".firebasedatabase.app"):
+		return classifyFirebaseDB(&s)
+	case strings.HasSuffix(host, ".appspot.com"):
+		return classifyAppspot(&s)
+	case strings.HasSuffix(host, ".r2.cloudflarestorage.com"):
+		return classifyR2(&s)
+	case strings.HasSuffix(host, ".r2.dev"):
+		return classifyR2Dev(&s)
+	case strings.HasSuffix(host, ".digitaloceanspaces.com"):
+		return classifyDigitalOceanSpaces(&s)
+	case strings.HasSuffix(host, ".ondigitalocean.com"):
+		return classifyDigitalOceanDB(&s)
+	case strings.HasSuffix(host, ".supabase.co"):
+		return classifySupabase(&s)
+	case strings.HasSuffix(host, ".supabase.com"):
+		return classifySupabasePooler(&s)
+	case strings.HasSuffix(host, ".customer-oci.com"):
+		return classifyOCIResource(&s)
+	case strings.HasSuffix(host, ".oraclecloud.com"):
+		return classifyOCIGeneric(&s)
+	}
+	return model.Match{}, false
+}
+
+// ShouldEnrich reports whether a non-matching hostname is worth a DNS/CNAME
+// lookup. It rejects wildcards, the certspotter unparsable placeholder, single
+// labels and obvious provider service apexes.
+func ShouldEnrich(host string) bool {
+	host = trimTrailingDots(host)
+	if len(host) < 3 || host[0] == '*' {
+		return false
+	}
+	if host[0] == '<' { // certspotter's "<unparsable>" placeholder
+		return false
+	}
+	if !strings.Contains(host, ".") {
+		return false
+	}
+	if hasUpper(host) {
+		host = strings.ToLower(host)
+	}
+	// Provider service apexes never yield a customer resource via CNAME.
+	switch {
+	case host == "amazonaws.com", strings.HasSuffix(host, ".amazonaws.com"):
+		return false
+	case host == "core.windows.net", strings.HasSuffix(host, ".windows.net"):
+		return false
+	case host == "azure.com", strings.HasSuffix(host, ".azure.com"):
+		return false
+	case host == "googleapis.com", strings.HasSuffix(host, ".googleapis.com"):
+		return false
+	case host == "cloudflarestorage.com", host == "r2.dev":
+		return false
+	case strings.HasSuffix(host, ".digitaloceanspaces.com"):
+		return false
+	case strings.HasSuffix(host, ".supabase.co"), strings.HasSuffix(host, ".supabase.com"):
+		return false
+	case strings.HasSuffix(host, ".oraclecloud.com"), strings.HasSuffix(host, ".customer-oci.com"):
+		return false
+	}
+	return true
+}

internal/classify/classify_test.go

@@ -0,0 +1,243 @@
+package classify
+
+import (
+	"strings"
+	"testing"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+type want struct {
+	provider         model.Provider
+	service          model.Service
+	resource         string
+	region           string
+	resourceSpecific bool
+	semantic         model.PublicSemantic
+}
+
+func run(t *testing.T, host string, w *want) {
+	t.Helper()
+	m, ok := Classify(host)
+	if w == nil {
+		if ok {
+			t.Fatalf("Classify(%q) matched %+v, want no match", host, m)
+		}
+		return
+	}
+	if !ok {
+		t.Fatalf("Classify(%q) did not match, want %+v", host, *w)
+	}
+	if m.Provider != w.provider || m.Service != w.service {
+		t.Fatalf("Classify(%q) = %s/%s, want %s/%s", host, m.Provider, m.Service, w.provider, w.service)
+	}
+	if w.resource != "" && m.Resource != w.resource {
+		t.Fatalf("Classify(%q) resource = %q, want %q", host, m.Resource, w.resource)
+	}
+	if w.region != "" && m.Region != w.region {
+		t.Fatalf("Classify(%q) region = %q, want %q", host, m.Region, w.region)
+	}
+	if m.ResourceSpecific != w.resourceSpecific {
+		t.Fatalf("Classify(%q) resourceSpecific = %v, want %v", host, m.ResourceSpecific, w.resourceSpecific)
+	}
+	if w.semantic != 0 && m.PublicSemantic != w.semantic {
+		t.Fatalf("Classify(%q) semantic = %s, want %s", host, m.PublicSemantic, w.semantic)
+	}
+}
+
+func TestClassifyMatches(t *testing.T) {
+	cases := []struct {
+		host string
+		w    want
+	}{
+		// AWS S3
+		{"bucket.s3.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+		{"bucket.s3.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3.dualstack.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3-website.eu-west-2.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-2", true, 0}},
+		{"bucket.s3-website-us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+		{"ap-123456.s3-accesspoint.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "ap-123456", "us-east-1", true, 0}},
+		{"x.s3-accesspoint.dualstack.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
+		{"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}},
+		{"x.mrap.accesspoint.s3-global.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "", true, 0}},
+		{"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}},
+		// AWS databases
+		{"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.proxy-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}},
+		{"mydb.cluster-abc.us-east-1.docdb.amazonaws.com", want{model.ProviderAWS, model.ServiceDocumentDB, "mydb", "us-east-1", true, 0}},
+		{"cluster.id.eu-central-1.redshift.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
+		{"cluster.id.eu-central-1.redshift-serverless.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}},
+		{"search-example-abc.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "search-example-abc", "eu-west-1", true, model.PublicSemanticPublicEndpoint}},
+		{"vpc-example.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "vpc-example", "eu-west-1", true, model.PublicSemanticVPCEndpoint}},
+		{"mydb.cluster-abc.us-east-1.neptune.amazonaws.com", want{model.ProviderAWS, model.ServiceNeptune, "mydb", "us-east-1", true, 0}},
+		{"clustercfg.my.xxxx.memorydb.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceMemoryDB, "clustercfg", "us-east-1", true, 0}},
+		{"mycluster.abc.0001.use1.cache.amazonaws.com", want{model.ProviderAWS, model.ServiceElastiCache, "mycluster", "use1", true, 0}},
+		// Azure
+		{"myacct.blob.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
+		{"myacct.dfs.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}},
+		{"myacct.file.core.windows.net", want{model.ProviderAzure, model.ServiceFileStorage, "myacct", "", true, 0}},
+		{"myacct.queue.core.windows.net", want{model.ProviderAzure, model.ServiceQueue, "myacct", "", true, 0}},
+		{"myacct.table.core.windows.net", want{model.ProviderAzure, model.ServiceTableStorage, "myacct", "", true, 0}},
+		{"myacct.web.core.windows.net", want{model.ProviderAzure, model.ServiceWebApp, "myacct", "", true, 0}},
+		{"myserver.database.windows.net", want{model.ProviderAzure, model.ServiceDatabase, "myserver", "", true, 0}},
+		{"myserver.postgres.database.azure.com", want{model.ProviderAzure, model.ServicePostgres, "myserver", "", true, 0}},
+		{"myserver.mysql.database.azure.com", want{model.ProviderAzure, model.ServiceMySQL, "myserver", "", true, 0}},
+		{"mycosmos.documents.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
+		{"mycosmos.mongo.cosmos.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}},
+		{"myredis.redis.cache.windows.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
+		{"myredis.redis.azure.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}},
+		{"mysearch.search.windows.net", want{model.ProviderAzure, model.ServiceSearch, "mysearch", "", true, 0}},
+		// GCP
+		{"bucket.storage.googleapis.com", want{model.ProviderGCP, model.ServiceObjectStorage, "bucket", "", true, 0}},
+		{"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}},
+		{"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}},
+		{"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}},
+		// Cloudflare
+		{"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}},
+		{"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}},
+		// DigitalOcean
+		{"bucket.fra1.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
+		{"bucket.fra1.cdn.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}},
+		{"mydb.db.ondigitalocean.com", want{model.ProviderDigitalOcean, model.ServiceDatabase, "mydb", "", true, 0}},
+		// Supabase
+		{"abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServiceDatabase, "abcdefgh", "", true, 0}},
+		{"db.abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServicePostgres, "abcdefgh", "", true, 0}},
+		{"aws-0-us-east-1.pooler.supabase.com", want{model.ProviderSupabase, model.ServicePostgres, "aws-0-us-east-1", "", true, 0}},
+		// OCI
+		{"myns.objectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
+		{"myns.swiftobjectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}},
+		{"objectstorage.us-ashburn-1.oraclecloud.com", want{model.ProviderOracle, model.ServiceObjectStorage, "", "us-ashburn-1", false, 0}},
+		// normalization
+		{"BUCKET.S3.EU-CENTRAL-1.AMAZONAWS.COM", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}},
+		{"bucket.s3.amazonaws.com.", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}},
+	}
+	for _, c := range cases {
+		w := c.w
+		run(t, c.host, &w)
+	}
+}
+
+func TestClassifyRejects(t *testing.T) {
+	rejects := []string{
+		// AWS generic endpoints / non-resources
+		"amazonaws.com",
+		"s3.amazonaws.com",
+		"s3.eu-central-1.amazonaws.com",
+		"s3.dualstack.eu-central-1.amazonaws.com",
+		"s3-accesspoint.us-east-1.amazonaws.com",
+		"dynamodb.eu-central-1.amazonaws.com",
+		"redshift.eu-central-1.amazonaws.com",
+		"rds.us-east-1.amazonaws.com",
+		"docdb.us-east-1.amazonaws.com",
+		"neptune.us-east-1.amazonaws.com",
+		"es.us-east-1.amazonaws.com",
+		"memorydb.us-east-1.amazonaws.com",
+		"cache.amazonaws.com",
+		"use1.cache.amazonaws.com",
+		"s3-global.amazonaws.com",
+		// Azure bare service endpoints
+		"blob.core.windows.net",
+		"database.windows.net",
+		"postgres.database.azure.com",
+		"documents.azure.com",
+		"mongo.cosmos.azure.com",
+		"redis.cache.windows.net",
+		"redis.azure.net",
+		"search.windows.net",
+		// GCP bare service endpoints
+		"storage.googleapis.com",
+		"firebaseio.com",
+		"firebasedatabase.app",
+		"appspot.com",
+		// Cloudflare
+		"r2.cloudflarestorage.com",
+		"r2.dev",
+		// DigitalOcean
+		"fra1.digitaloceanspaces.com",
+		"db.ondigitalocean.com",
+		// Supabase
+		"supabase.co",
+		"pooler.supabase.com",
+		// OCI
+		"oraclecloud.com",
+		"customer-oci.com",
+		// wildcards
+		"*.example.com",
+		"*.blob.core.windows.net",
+		"*.s3.amazonaws.com",
+		// normal hostnames
+		"www.example.com",
+		"foo.bar.example.org",
+		"localhost",
+		"",
+		"-",
+		// invalid / too long
+		"foo bar.com",
+		strings.Repeat("a.", 130) + "com",
+	}
+	for _, host := range rejects {
+		run(t, host, nil)
+	}
+}
+
+func TestShouldEnrich(t *testing.T) {
+	yes := []string{"files.example.com", "db.example.com", "a.b.c.example.org"}
+	no := []string{"*.example.com", "s3.amazonaws.com", "foo.s3.amazonaws.com", "storage.googleapis.com", "<unparsable>", ""}
+	for _, h := range yes {
+		if !ShouldEnrich(h) {
+			t.Errorf("ShouldEnrich(%q) = false, want true", h)
+		}
+	}
+	for _, h := range no {
+		if ShouldEnrich(h) {
+			t.Errorf("ShouldEnrich(%q) = true, want false", h)
+		}
+	}
+}
+
+var benchInputs = []string{
+	"www.example.com",
+	"foo.bar.example.org",
+	"bucket.s3.eu-central-1.amazonaws.com",
+	"foo.blob.core.windows.net",
+	"db.example.supabase.co",
+	"project.firebaseio.com",
+	"search-example-abc.eu-west-1.es.amazonaws.com",
+}
+
+func BenchmarkClassifyAWS(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("bucket.s3.eu-central-1.amazonaws.com")
+	}
+}
+
+func BenchmarkClassifyAzure(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("foo.blob.core.windows.net")
+	}
+}
+
+func BenchmarkRejectNormalDomain(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Classify("www.example.com")
+	}
+}
+
+func BenchmarkClassifyMixed(b *testing.B) {
+	allocs := testing.AllocsPerRun(1000, func() {
+		for _, host := range benchInputs {
+			Classify(host)
+		}
+	})
+	if allocs != 0 {
+		b.Fatalf("Classify allocates %.2f allocs/op, want 0", allocs)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, host := range benchInputs {
+			Classify(host)
+		}
+	}
+}

internal/classify/cloudflare.go

@@ -0,0 +1,34 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyR2 handles <account-id>.r2.cloudflarestorage.com.
+func classifyR2(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "cloudflarestorage" || s.label(n-3) != "r2" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderCloudflare,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}
+
+// classifyR2Dev handles <account>.r2.dev.  The hostname form is explicitly a
+// public development/public URL, which is why it carries PublicSemanticPublicURL
+// while still not asserting that data is readable.
+func classifyR2Dev(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "dev" || s.label(n-2) != "r2" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderCloudflare,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		PublicSemantic:   model.PublicSemanticPublicURL,
+		Resource:         s.label(0),
+	}, true
+}

internal/classify/digitalocean.go

@@ -0,0 +1,39 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyDigitalOceanSpaces handles
+// <bucket>.<region>.digitaloceanspaces.com and the cdn variant.
+func classifyDigitalOceanSpaces(s *scanner) (model.Match, bool) {
+	n := s.n
+	if s.label(n-1) != "com" || s.label(n-2) != "digitaloceanspaces" {
+		return model.Match{}, false
+	}
+	// <bucket>.<region>.digitaloceanspaces.com (n==4) or the cdn variant (n==5)
+	if n != 4 && !(n == 5 && s.label(2) == "cdn") {
+		return model.Match{}, false
+	}
+	bucket := s.label(0)
+	region := s.label(1)
+	return model.Match{
+		Provider:         model.ProviderDigitalOcean,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         bucket,
+	}, true
+}
+
+// classifyDigitalOceanDB handles <name>.db.ondigitalocean.com.
+func classifyDigitalOceanDB(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "ondigitalocean" || s.label(n-3) != "db" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderDigitalOcean,
+		Service:          model.ServiceDatabase,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}

internal/classify/gcp.go

@@ -0,0 +1,49 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+func gcp(service model.Service, resource, region string) model.Match {
+	return model.Match{
+		Provider:         model.ProviderGCP,
+		Service:          service,
+		ResourceSpecific: true,
+		Region:           region,
+		Resource:         resource,
+	}
+}
+
+// classifyGCS handles <bucket>.storage.googleapis.com.
+func classifyGCS(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "googleapis" || s.label(n-3) != "storage" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceObjectStorage, s.label(0), ""), true
+}
+
+// classifyFirebaseIO handles <database>.firebaseio.com.
+func classifyFirebaseIO(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "firebaseio" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceFirebaseDB, s.label(0), ""), true
+}
+
+// classifyFirebaseDB handles <database>.<region>.firebasedatabase.app.
+func classifyFirebaseDB(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 4 || s.label(n-1) != "app" || s.label(n-2) != "firebasedatabase" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceFirebaseDB, s.label(0), s.label(1)), true
+}
+
+// classifyAppspot handles <project>.appspot.com.
+func classifyAppspot(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "appspot" {
+		return model.Match{}, false
+	}
+	return gcp(model.ServiceWebApp, s.label(0), ""), true
+}

internal/classify/oracle.go

@@ -0,0 +1,44 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifyOCIResource handles
+// <namespace>.objectstorage.<region>.oci.customer-oci.com and the swift variant.
+// The namespace is in the hostname, so it is resource specific.
+func classifyOCIResource(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n < 6 || s.label(n-1) != "com" || s.label(n-2) != "customer-oci" || s.label(n-3) != "oci" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 5)
+	if svc != "objectstorage" && svc != "swiftobjectstorage" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderOracle,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: true,
+		Region:           s.label(n - 4),
+		Resource:         s.label(0),
+	}, true
+}
+
+// classifyOCIGeneric handles objectstorage.<region>.oraclecloud.com. The bucket
+// name lives in the URL path rather than the hostname, so resource_specific is
+// false and these normally do not become findings.
+func classifyOCIGeneric(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "oraclecloud" {
+		return model.Match{}, false
+	}
+	svc := s.label(n - 4)
+	if svc != "objectstorage" && svc != "swiftobjectstorage" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderOracle,
+		Service:          model.ServiceObjectStorage,
+		ResourceSpecific: false,
+		Region:           s.label(n - 3),
+	}, true
+}

internal/classify/supabase.go

@@ -0,0 +1,45 @@
+package classify
+
+import "tea.karlcom.de/karl/bucketeer/internal/model"
+
+// classifySupabase handles <project-ref>.supabase.co and
+// db.<project-ref>.supabase.co (the latter classified as PostgreSQL).
+func classifySupabase(s *scanner) (model.Match, bool) {
+	n := s.n
+	if s.label(n-1) != "co" || s.label(n-2) != "supabase" {
+		return model.Match{}, false
+	}
+	switch n {
+	case 3:
+		return model.Match{
+			Provider:         model.ProviderSupabase,
+			Service:          model.ServiceDatabase,
+			ResourceSpecific: true,
+			Resource:         s.label(0),
+		}, true
+	case 4:
+		if s.label(0) == "db" {
+			return model.Match{
+				Provider:         model.ProviderSupabase,
+				Service:          model.ServicePostgres,
+				ResourceSpecific: true,
+				Resource:         s.label(1),
+			}, true
+		}
+	}
+	return model.Match{}, false
+}
+
+// classifySupabasePooler handles aws-<index>-<region>.pooler.supabase.com.
+func classifySupabasePooler(s *scanner) (model.Match, bool) {
+	n := s.n
+	if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "supabase" || s.label(n-3) != "pooler" {
+		return model.Match{}, false
+	}
+	return model.Match{
+		Provider:         model.ProviderSupabase,
+		Service:          model.ServicePostgres,
+		ResourceSpecific: true,
+		Resource:         s.label(0),
+	}, true
+}

internal/config/config.go

@@ -0,0 +1,160 @@
+// Package config loads all ctcloud configuration from environment variables.
+// There are no configuration files at runtime.
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Config struct {
+	DatabaseURL string
+	DBMaxConns  int
+
+	DNSServers       []string
+	DNSWorkers       int
+	DNSTimeout       time.Duration
+	DNSCacheSize     int
+	DNSMaxCNAMEdepth int
+	DNSPositiveTTL   time.Duration
+	DNSNegativeTTL   time.Duration
+
+	DirectQueueSize int
+	DNSQueueSize    int
+	DBQueueSize     int
+
+	DBBatchSize     int
+	DBFlushInterval time.Duration
+
+	ActiveProbing bool
+	ProbeWorkers  int
+	ProbeTimeout  time.Duration
+
+	MetricsAddr   string
+	LogLevel      string
+	StatsInterval time.Duration
+
+	LogListSource   string
+	StartAtEnd      bool
+	ShutdownTimeout time.Duration
+	DedupeTTL       time.Duration
+}
+
+// DefaultLogListSource is the public CT log list used by certspotter.
+const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.json"
+
+// DefaultDNSServers are load-balanced across. Set DNS_SERVERS=system to use the
+// host resolver instead, or provide your own comma-separated list.
+var DefaultDNSServers = []string{"1.1.1.1:53", "8.8.8.8:53", "1.0.0.1:53"}
+
+func Load() (*Config, error) {
+	c := &Config{
+		DatabaseURL:      os.Getenv("DATABASE_URL"),
+		DBMaxConns:       envInt("DB_MAX_CONNS", 8),
+		DNSServers:       envList("DNS_SERVERS", DefaultDNSServers),
+		DNSWorkers:       envInt("DNS_WORKERS", 16),
+		DNSTimeout:       envDuration("DNS_TIMEOUT", 5*time.Second),
+		DNSCacheSize:     envInt("DNS_CACHE_SIZE", 1<<20),
+		DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8),
+		DNSPositiveTTL:   envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
+		DNSNegativeTTL:   envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
+		DirectQueueSize:  envInt("DIRECT_QUEUE_SIZE", 8192),
+		DNSQueueSize:     envInt("DNS_QUEUE_SIZE", 65536),
+		DBQueueSize:      envInt("DB_QUEUE_SIZE", 8192),
+		DBBatchSize:      envInt("DB_BATCH_SIZE", 500),
+		DBFlushInterval:  envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
+		ActiveProbing:    envBool("ACTIVE_PROBING", false),
+		ProbeWorkers:     envInt("PROBE_WORKERS", 4),
+		ProbeTimeout:     envDuration("PROBE_TIMEOUT", 5*time.Second),
+		MetricsAddr:      envString("METRICS_ADDR", ":9090"),
+		LogLevel:         envString("LOG_LEVEL", "info"),
+		StatsInterval:    envDuration("STATS_INTERVAL", 60*time.Second),
+		LogListSource:    envString("LOG_LIST_SOURCE", DefaultLogListSource),
+		StartAtEnd:       envBool("START_AT_END", false),
+		ShutdownTimeout:  envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
+		DedupeTTL:        envDuration("DEDUP_TTL", time.Hour),
+	}
+	if err := c.validate(); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+func (c *Config) validate() error {
+	if c.DatabaseURL == "" {
+		return fmt.Errorf("DATABASE_URL is required")
+	}
+	if c.DNSWorkers < 1 {
+		return fmt.Errorf("DNS_WORKERS must be >= 1")
+	}
+	if c.DNSMaxCNAMEdepth < 1 {
+		return fmt.Errorf("DNS_MAX_CNAME_DEPTH must be >= 1")
+	}
+	if c.DirectQueueSize < 1 || c.DNSQueueSize < 1 || c.DBQueueSize < 1 {
+		return fmt.Errorf("queue sizes must be >= 1")
+	}
+	if c.DBBatchSize < 1 {
+		return fmt.Errorf("DB_BATCH_SIZE must be >= 1")
+	}
+	switch c.LogLevel {
+	case "debug", "info", "warn", "error":
+	default:
+		return fmt.Errorf("LOG_LEVEL must be one of debug, info, warn, error")
+	}
+	return nil
+}
+
+func envList(key string, def []string) []string {
+	v := os.Getenv(key)
+	if v == "" {
+		return def
+	}
+	parts := strings.Split(v, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	if len(out) == 0 {
+		return def
+	}
+	return out
+}
+
+func envString(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func envInt(key string, def int) int {
+	if v := os.Getenv(key); v != "" {
+		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+			return n
+		}
+	}
+	return def
+}
+
+func envBool(key string, def bool) bool {
+	if v := os.Getenv(key); v != "" {
+		if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+			return b
+		}
+	}
+	return def
+}
+
+func envDuration(key string, def time.Duration) time.Duration {
+	if v := os.Getenv(key); v != "" {
+		if d, err := time.ParseDuration(strings.TrimSpace(v)); err == nil {
+			return d
+		}
+	}
+	return def
+}

internal/config/config_test.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+func TestLoadDefaults(t *testing.T) {
+	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	c, err := Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.DatabaseURL != "postgres://localhost/ctcloud" {
+		t.Fatalf("database url = %q", c.DatabaseURL)
+	}
+	if c.DNSWorkers != 16 || c.DNSMaxCNAMEdepth != 8 {
+		t.Fatalf("unexpected DNS defaults: %+v", c)
+	}
+	if c.ActiveProbing {
+		t.Fatal("active probing must default to false")
+	}
+}
+
+func TestLoadOverrides(t *testing.T) {
+	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("DNS_WORKERS", "4")
+	t.Setenv("ACTIVE_PROBING", "true")
+	t.Setenv("DB_FLUSH_INTERVAL", "250ms")
+	t.Setenv("STATS_INTERVAL", "30s")
+	c, err := Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.DNSWorkers != 4 || !c.ActiveProbing {
+		t.Fatalf("overrides not applied: %+v", c)
+	}
+	if c.DBFlushInterval != 250*time.Millisecond || c.StatsInterval != 30*time.Second {
+		t.Fatalf("duration overrides not applied: %+v", c)
+	}
+}
+
+func TestLoadInvalid(t *testing.T) {
+	t.Setenv("DATABASE_URL", "")
+	if _, err := Load(); err == nil {
+		t.Fatal("expected error without DATABASE_URL")
+	}
+	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("LOG_LEVEL", "verbose")
+	if _, err := Load(); err == nil {
+		t.Fatal("expected error for bad LOG_LEVEL")
+	}
+}

internal/dns/cache.go

@@ -0,0 +1,99 @@
+package dns
+
+import (
+	"sync"
+	"time"
+)
+
+type cacheEntry[V any] struct {
+	val V
+	exp time.Time
+}
+
+type shard[V any] struct {
+	mu sync.Mutex
+	m  map[string]cacheEntry[V]
+}
+
+// TTLCache is a sharded, memory-bounded, in-memory TTL cache. It is used for
+// positive DNS results, negative DNS results, CNAME targets and hostname dedup.
+//
+// ponytail: once a shard is full, an arbitrary entry is evicted rather than the
+// least-recently-used one. Swap in an LRU only if the hit rate actually matters.
+type TTLCache[V any] struct {
+	shards   []shard[V]
+	mask     uint64
+	maxShard int
+}
+
+func NewTTLCache[V any](maxEntries, shards int) *TTLCache[V] {
+	if shards < 1 {
+		shards = 1
+	}
+	n := 1
+	for n < shards {
+		n <<= 1
+	}
+	maxShard := maxEntries / n
+	if maxShard < 1 {
+		maxShard = 1
+	}
+	c := &TTLCache[V]{mask: uint64(n - 1), maxShard: maxShard}
+	c.shards = make([]shard[V], n)
+	for i := range c.shards {
+		c.shards[i].m = make(map[string]cacheEntry[V], maxShard)
+	}
+	return c
+}
+
+func hashString(s string) uint64 {
+	const prime = 1099511628211
+	h := uint64(14695981039346656037)
+	for i := 0; i < len(s); i++ {
+		h ^= uint64(s[i])
+		h *= prime
+	}
+	return h
+}
+
+func (c *TTLCache[V]) Get(key string) (V, bool) {
+	s := &c.shards[hashString(key)&c.mask]
+	s.mu.Lock()
+	e, ok := s.m[key]
+	if ok && time.Now().Before(e.exp) {
+		s.mu.Unlock()
+		return e.val, true
+	}
+	if ok {
+		delete(s.m, key)
+	}
+	s.mu.Unlock()
+	var zero V
+	return zero, false
+}
+
+func (c *TTLCache[V]) Set(key string, val V, ttl time.Duration) {
+	if ttl <= 0 {
+		return
+	}
+	s := &c.shards[hashString(key)&c.mask]
+	s.mu.Lock()
+	if len(s.m) >= c.maxShard {
+		for k := range s.m {
+			delete(s.m, k)
+			break
+		}
+	}
+	s.m[key] = cacheEntry[V]{val: val, exp: time.Now().Add(ttl)}
+	s.mu.Unlock()
+}
+
+func (c *TTLCache[V]) Len() int {
+	total := 0
+	for i := range c.shards {
+		c.shards[i].mu.Lock()
+		total += len(c.shards[i].m)
+		c.shards[i].mu.Unlock()
+	}
+	return total
+}

internal/dns/pool.go

@@ -0,0 +1,119 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"sync/atomic"
+	"time"
+)
+
+// ServerPool load-balances DNS lookups across multiple resolvers using
+// round-robin, and fails over to the next server on a timeout or server error.
+// Definitive answers (NXDOMAIN) are returned immediately and are not retried on
+// every server.
+// defaultAttemptTimeout bounds each individual server attempt so a dead
+// resolver costs a couple of seconds before failover, not the entire lookup
+// budget.
+const defaultAttemptTimeout = 2 * time.Second
+
+type ServerPool struct {
+	resolvers      []Resolver
+	next           atomic.Uint64
+	attemptTimeout time.Duration
+}
+
+func (p *ServerPool) withAttemptTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
+	t := p.attemptTimeout
+	if t <= 0 {
+		t = defaultAttemptTimeout
+	}
+	return context.WithTimeout(ctx, t)
+}
+
+// NewServerPool builds a pool from "host", "host:port" or "system" entries.
+// An empty list uses the system resolver.
+func NewServerPool(servers []string) *ServerPool {
+	var resolvers []Resolver
+	for _, s := range servers {
+		s = strings.TrimSpace(s)
+		switch {
+		case s == "":
+			continue
+		case s == "system" || s == "local":
+			return &ServerPool{resolvers: []Resolver{&GoResolver{R: net.DefaultResolver}}}
+		default:
+			if !strings.Contains(s, ":") {
+				s += ":53"
+			}
+			resolvers = append(resolvers, &GoResolver{R: newServerResolver(s)})
+		}
+	}
+	if len(resolvers) == 0 {
+		resolvers = append(resolvers, &GoResolver{R: net.DefaultResolver})
+	}
+	return &ServerPool{resolvers: resolvers}
+}
+
+func newServerResolver(server string) *net.Resolver {
+	return &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
+			if network != "tcp" {
+				network = "udp"
+			}
+			d := net.Dialer{Timeout: 5 * time.Second}
+			return d.DialContext(ctx, network, server)
+		},
+	}
+}
+
+func (p *ServerPool) pick() int {
+	return int((p.next.Add(1) - 1) % uint64(len(p.resolvers)))
+}
+
+// definitive reports whether err is a real DNS answer (NXDOMAIN) that should not
+// be retried against other servers.
+func definitive(err error) bool {
+	var dnsErr *net.DNSError
+	return errors.As(err, &dnsErr) && dnsErr.IsNotFound
+}
+
+func (p *ServerPool) LookupCNAME(ctx context.Context, host string) (string, error) {
+	n := len(p.resolvers)
+	start := p.pick()
+	var last error
+	for i := 0; i < n; i++ {
+		actx, cancel := p.withAttemptTimeout(ctx)
+		cname, err := p.resolvers[(start+i)%n].LookupCNAME(actx, host)
+		cancel()
+		if err == nil {
+			return cname, nil
+		}
+		if definitive(err) {
+			return "", err
+		}
+		last = err
+	}
+	return "", last
+}
+
+func (p *ServerPool) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	n := len(p.resolvers)
+	start := p.pick()
+	var last error
+	for i := 0; i < n; i++ {
+		actx, cancel := p.withAttemptTimeout(ctx)
+		ips, err := p.resolvers[(start+i)%n].LookupIP(actx, host)
+		cancel()
+		if err == nil {
+			return ips, nil
+		}
+		if definitive(err) {
+			return nil, err
+		}
+		last = err
+	}
+	return nil, last
+}

internal/dns/pool_test.go

@@ -0,0 +1,68 @@
+package dns
+
+import (
+	"context"
+	"net"
+	"sync/atomic"
+	"testing"
+)
+
+type fakeLookup struct {
+	calls    atomic.Int32
+	cnameErr error
+	ipErr    error
+}
+
+func (f *fakeLookup) LookupCNAME(ctx context.Context, host string) (string, error) {
+	f.calls.Add(1)
+	if f.cnameErr != nil {
+		return "", f.cnameErr
+	}
+	return host, nil
+}
+
+func (f *fakeLookup) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	f.calls.Add(1)
+	if f.ipErr != nil {
+		return nil, f.ipErr
+	}
+	return []net.IP{net.ParseIP("1.2.3.4")}, nil
+}
+
+func TestServerPoolRoundRobin(t *testing.T) {
+	a, b := &fakeLookup{}, &fakeLookup{}
+	p := &ServerPool{resolvers: []Resolver{a, b}}
+	for i := 0; i < 4; i++ {
+		if _, err := p.LookupIP(context.Background(), "example.com"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if a.calls.Load() != 2 || b.calls.Load() != 2 {
+		t.Fatalf("round robin calls: a=%d b=%d, want 2/2", a.calls.Load(), b.calls.Load())
+	}
+}
+
+func TestServerPoolFailover(t *testing.T) {
+	a := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}}
+	b := &fakeLookup{}
+	p := &ServerPool{resolvers: []Resolver{a, b}}
+	ips, err := p.LookupIP(context.Background(), "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ips) != 1 || a.calls.Load() != 1 {
+		t.Fatalf("expected failover to second server, a.calls=%d", a.calls.Load())
+	}
+}
+
+func TestServerPoolNoFailoverOnNXDOMAIN(t *testing.T) {
+	a := &fakeLookup{ipErr: &net.DNSError{Err: "no such host", IsNotFound: true}}
+	b := &fakeLookup{}
+	p := &ServerPool{resolvers: []Resolver{a, b}}
+	if _, err := p.LookupIP(context.Background(), "missing.example.com"); err == nil {
+		t.Fatal("expected error")
+	}
+	if b.calls.Load() != 0 {
+		t.Fatalf("NXDOMAIN must not fail over; b.calls=%d", b.calls.Load())
+	}
+}

internal/dns/resolver.go

@@ -0,0 +1,131 @@
+// Package dns provides a bounded, cached DNS enricher that follows CNAME chains
+// with loop detection.
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+)
+
+// ErrCNAMELoop is returned when a CNAME chain revisits a host.
+var ErrCNAMELoop = errors.New("cname loop detected")
+
+// Resolver is the subset of DNS operations the enricher needs.
+type Resolver interface {
+	LookupCNAME(ctx context.Context, host string) (string, error)
+	LookupIP(ctx context.Context, host string) ([]net.IP, error)
+}
+
+// Result is a resolved hostname: its CNAME chain (chain[0] is the queried host)
+// and any A/AAAA addresses.
+type Result struct {
+	Chain []string
+	IPs   []net.IP
+}
+
+// GoResolver uses the standard library resolver.
+//
+// ponytail: net.Resolver.LookupCNAME returns the final canonical name, so the
+// chain has at most two entries and intermediate CNAME hops are not observed.
+// Swap in miekg/dns if the full chain is ever needed.
+type GoResolver struct {
+	R *net.Resolver
+}
+
+func NewGoResolver() *GoResolver {
+	return &GoResolver{R: net.DefaultResolver}
+}
+
+func (g *GoResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	return g.R.LookupCNAME(ctx, host)
+}
+
+func (g *GoResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	return g.R.LookupIP(ctx, "ip", host)
+}
+
+// Follow walks a CNAME chain up to maxDepth hops, detecting loops.
+func Follow(ctx context.Context, r Resolver, host string, maxDepth int) ([]string, error) {
+	if maxDepth < 1 {
+		maxDepth = 1
+	}
+	chain := []string{host}
+	cur := host
+	for i := 0; i < maxDepth; i++ {
+		next, err := r.LookupCNAME(ctx, cur)
+		if err != nil {
+			return chain, err
+		}
+		next = strings.TrimSuffix(next, ".")
+		if next == "" || next == cur {
+			break
+		}
+		for _, seen := range chain {
+			if seen == next {
+				return chain, ErrCNAMELoop
+			}
+		}
+		chain = append(chain, next)
+		cur = next
+	}
+	return chain, nil
+}
+
+// Enricher resolves hostnames with caching and bounded CNAME depth. It is safe
+// for concurrent use.
+type Enricher struct {
+	Resolver    Resolver
+	Cache       *TTLCache[Result]
+	PositiveTTL time.Duration
+	NegativeTTL time.Duration
+	Timeout     time.Duration
+	MaxDepth    int
+	Metrics     *metrics.Metrics
+}
+
+// Resolve returns the result for host. ok is false when the lookup failed;
+// failures (including "no such host") are negatively cached for NegativeTTL.
+func (e *Enricher) Resolve(ctx context.Context, host string) (Result, bool) {
+	if e.Timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, e.Timeout)
+		defer cancel()
+	}
+	if e.Cache != nil {
+		if res, ok := e.Cache.Get(host); ok {
+			if e.Metrics != nil {
+				e.Metrics.DNSCacheHits.Add(1)
+			}
+			return res, len(res.Chain) > 0 || len(res.IPs) > 0
+		}
+	}
+	if e.Metrics != nil {
+		e.Metrics.DNSQueries.Add(1)
+	}
+
+	chain, cnameErr := Follow(ctx, e.Resolver, host, e.MaxDepth)
+	if cnameErr != nil && len(chain) == 1 {
+		// A definitive NXDOMAIN has no A/AAAA to fetch either, so skip the
+		// second lookup. NXDOMAIN is a valid answer, not an infrastructure error.
+		if e.Metrics != nil && !definitive(cnameErr) {
+			e.Metrics.DNSErrors.Add(1)
+		}
+		if e.Cache != nil {
+			e.Cache.Set(host, Result{}, e.NegativeTTL)
+		}
+		return Result{}, false
+	}
+	res := Result{Chain: chain}
+	if ips, err := e.Resolver.LookupIP(ctx, host); err == nil {
+		res.IPs = ips
+	}
+	if e.Cache != nil {
+		e.Cache.Set(host, res, e.PositiveTTL)
+	}
+	return res, true
+}

internal/dns/resolver_test.go

@@ -0,0 +1,142 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"testing"
+	"time"
+)
+
+type fakeResolver struct {
+	cnames map[string]string
+	ips    map[string][]net.IP
+	err    error
+}
+
+func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	if f.err != nil {
+		return "", f.err
+	}
+	if c, ok := f.cnames[host]; ok {
+		return c, nil
+	}
+	return host, nil
+}
+
+func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	if f.err != nil {
+		return nil, f.err
+	}
+	return f.ips[host], nil
+}
+
+func TestFollowChain(t *testing.T) {
+	r := &fakeResolver{cnames: map[string]string{
+		"a.example.com":      "b.example.com",
+		"b.example.com":      "c.s3.amazonaws.com",
+		"c.s3.amazonaws.com": "c.s3.amazonaws.com",
+	}}
+	chain, err := Follow(context.Background(), r, "a.example.com", 8)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"a.example.com", "b.example.com", "c.s3.amazonaws.com"}
+	if len(chain) != len(want) {
+		t.Fatalf("chain = %v, want %v", chain, want)
+	}
+	for i := range want {
+		if chain[i] != want[i] {
+			t.Fatalf("chain = %v, want %v", chain, want)
+		}
+	}
+}
+
+func TestFollowLoop(t *testing.T) {
+	r := &fakeResolver{cnames: map[string]string{
+		"a.example.com": "b.example.com",
+		"b.example.com": "a.example.com",
+	}}
+	chain, err := Follow(context.Background(), r, "a.example.com", 8)
+	if !errors.Is(err, ErrCNAMELoop) {
+		t.Fatalf("err = %v, want ErrCNAMELoop", err)
+	}
+	if len(chain) < 2 {
+		t.Fatalf("chain = %v, want at least 2 entries", chain)
+	}
+}
+
+func TestFollowDepth(t *testing.T) {
+	r := &fakeResolver{cnames: map[string]string{
+		"h1.example.com": "h2.example.com",
+		"h2.example.com": "h3.example.com",
+		"h3.example.com": "h4.example.com",
+	}}
+	chain, err := Follow(context.Background(), r, "h1.example.com", 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(chain) != 3 {
+		t.Fatalf("chain = %v, want 3 entries (host + 2 hops)", chain)
+	}
+}
+
+func TestEnricherCaches(t *testing.T) {
+	r := &fakeResolver{
+		cnames: map[string]string{"a.example.com": "b.s3.amazonaws.com"},
+		ips:    map[string][]net.IP{"a.example.com": {net.ParseIP("1.2.3.4")}},
+	}
+	e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
+	if _, ok := e.Resolve(context.Background(), "a.example.com"); !ok {
+		t.Fatal("first resolve failed")
+	}
+	r.cnames = nil // a cached lookup must not hit the resolver
+	res, ok := e.Resolve(context.Background(), "a.example.com")
+	if !ok || len(res.IPs) != 1 {
+		t.Fatalf("cached resolve = %+v ok=%v", res, ok)
+	}
+}
+
+func TestEnricherNegativeCache(t *testing.T) {
+	r := &fakeResolver{err: &net.DNSError{Err: "no such host", IsNotFound: true}}
+	e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
+	if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok {
+		t.Fatal("expected failure")
+	}
+	if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok {
+		t.Fatal("expected cached failure")
+	}
+}
+
+type ctxResolver struct{}
+
+func (ctxResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	return "", ctx.Err()
+}
+func (ctxResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	return nil, ctx.Err()
+}
+
+func TestResolveContextCancellation(t *testing.T) {
+	e := &Enricher{Resolver: ctxResolver{}, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, ok := e.Resolve(ctx, "example.com"); ok {
+		t.Fatal("expected cancellation to fail resolution")
+	}
+}
+
+func TestCacheBoundAndExpiry(t *testing.T) {
+	c := NewTTLCache[int](8, 2)
+	for i := 0; i < 100; i++ {
+		c.Set(string(rune('a'+i%26))+string(rune('0'+i%10)), i, time.Minute)
+	}
+	if c.Len() > 8 {
+		t.Fatalf("cache len = %d, want <= 8", c.Len())
+	}
+	c.Set("k", 1, time.Millisecond)
+	time.Sleep(5 * time.Millisecond)
+	if _, ok := c.Get("k"); ok {
+		t.Fatal("expected expired entry to be gone")
+	}
+}

internal/metrics/metrics.go

@@ -0,0 +1,91 @@
+// Package metrics exposes lightweight Prometheus text-format metrics without
+// pulling in a metrics dependency. Counters are plain atomics; gauges are
+// callback functions so they always report current values.
+package metrics
+
+import (
+	"fmt"
+	"net/http"
+	"runtime"
+	"sync/atomic"
+)
+
+type gauge struct {
+	name  string
+	help  string
+	extra string
+	fn    func() float64
+}
+
+type Metrics struct {
+	CTCertificates atomic.Uint64
+	CTIdentifiers  atomic.Uint64
+
+	DirectMatches atomic.Uint64
+	CNAMEMatches  atomic.Uint64
+
+	DNSQueries   atomic.Uint64
+	DNSCacheHits atomic.Uint64
+	DNSErrors    atomic.Uint64
+
+	Candidates atomic.Uint64
+
+	DBRows   atomic.Uint64
+	DBErrors atomic.Uint64
+
+	QueueDrops    atomic.Uint64
+	QueueFull     atomic.Uint64
+	MalformedCT   atomic.Uint64
+	CTErrors      atomic.Uint64
+	ProbeAttempts atomic.Uint64
+	ProbePublic   atomic.Uint64
+
+	gauges []gauge
+}
+
+func New() *Metrics {
+	m := &Metrics{}
+	m.gauges = []gauge{
+		{"go_goroutines", "Number of goroutines", "", func() float64 { return float64(runtime.NumGoroutine()) }},
+		{"process_system_bytes", "Bytes of memory obtained from the OS", "", func() float64 {
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+			return float64(ms.Sys)
+		}},
+	}
+	return m
+}
+
+// SetQueueDepth registers a gauge reporting the live depth of a named queue.
+func (m *Metrics) SetQueueDepth(name string, fn func() float64) {
+	m.gauges = append(m.gauges, gauge{"queue_depth", "Current depth of an internal queue", `{queue="` + name + `"}`, fn})
+}
+
+// Handler returns an http.Handler serving the Prometheus text exposition format.
+func (m *Metrics) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+		counter := func(name, help string, v uint64) {
+			fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v)
+		}
+		counter("ct_certificates_total", "Certificates processed from CT logs", m.CTCertificates.Load())
+		counter("ct_identifiers_total", "DNS identifiers parsed from CT certificates", m.CTIdentifiers.Load())
+		counter("classifier_direct_matches_total", "Identifiers matching a provider directly", m.DirectMatches.Load())
+		counter("classifier_cname_matches_total", "CNAME targets matching a provider", m.CNAMEMatches.Load())
+		counter("dns_queries_total", "DNS queries performed", m.DNSQueries.Load())
+		counter("dns_cache_hits_total", "DNS lookups served from cache", m.DNSCacheHits.Load())
+		counter("dns_errors_total", "DNS lookup errors", m.DNSErrors.Load())
+		counter("candidates_total", "Provider candidates produced", m.Candidates.Load())
+		counter("db_rows_total", "Rows written to PostgreSQL", m.DBRows.Load())
+		counter("db_errors_total", "PostgreSQL write errors", m.DBErrors.Load())
+		counter("queue_drops_total", "Items dropped because a queue was full", m.QueueDrops.Load())
+		counter("queue_full_total", "Times a queue was observed full", m.QueueFull.Load())
+		counter("ct_malformed_entries_total", "Malformed CT log entries", m.MalformedCT.Load())
+		counter("ct_errors_total", "Non-fatal CT errors", m.CTErrors.Load())
+		counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
+		counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
+		for _, g := range m.gauges {
+			fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
+		}
+	})
+}

internal/model/model.go

@@ -0,0 +1,219 @@
+// Package model holds the compact enums and hot-path structs shared by the
+// classifier, pipeline and store.
+package model
+
+// Provider identifies a cloud/database/storage provider.
+type Provider uint8
+
+const (
+	ProviderUnknown Provider = iota
+	ProviderAWS
+	ProviderAzure
+	ProviderGCP
+	ProviderCloudflare
+	ProviderDigitalOcean
+	ProviderSupabase
+	ProviderOracle
+)
+
+func (p Provider) String() string {
+	switch p {
+	case ProviderAWS:
+		return "aws"
+	case ProviderAzure:
+		return "azure"
+	case ProviderGCP:
+		return "gcp"
+	case ProviderCloudflare:
+		return "cloudflare"
+	case ProviderDigitalOcean:
+		return "digitalocean"
+	case ProviderSupabase:
+		return "supabase"
+	case ProviderOracle:
+		return "oracle"
+	default:
+		return "unknown"
+	}
+}
+
+// Service identifies the provider service the resource belongs to.
+type Service uint8
+
+const (
+	ServiceUnknown Service = iota
+
+	ServiceObjectStorage
+	ServiceS3
+	ServiceBlobStorage
+	ServiceDatabase
+	ServicePostgres
+	ServiceMySQL
+	ServiceRDS
+	ServiceDocumentDB
+	ServiceRedshift
+	ServiceOpenSearch
+	ServiceNeptune
+	ServiceRedis
+	ServiceMemoryDB
+	ServiceCosmosDB
+	ServiceFirebaseDB
+	ServiceElastiCache
+	ServiceSearch
+	ServiceQueue
+	ServiceFileStorage
+	ServiceTableStorage
+	ServiceWebApp
+)
+
+func (s Service) String() string {
+	switch s {
+	case ServiceObjectStorage:
+		return "object_storage"
+	case ServiceS3:
+		return "s3"
+	case ServiceBlobStorage:
+		return "blob_storage"
+	case ServiceDatabase:
+		return "database"
+	case ServicePostgres:
+		return "postgres"
+	case ServiceMySQL:
+		return "mysql"
+	case ServiceRDS:
+		return "rds"
+	case ServiceDocumentDB:
+		return "documentdb"
+	case ServiceRedshift:
+		return "redshift"
+	case ServiceOpenSearch:
+		return "opensearch"
+	case ServiceNeptune:
+		return "neptune"
+	case ServiceRedis:
+		return "redis"
+	case ServiceMemoryDB:
+		return "memorydb"
+	case ServiceCosmosDB:
+		return "cosmosdb"
+	case ServiceFirebaseDB:
+		return "firebase_db"
+	case ServiceElastiCache:
+		return "elasticache"
+	case ServiceSearch:
+		return "search"
+	case ServiceQueue:
+		return "queue"
+	case ServiceFileStorage:
+		return "file_storage"
+	case ServiceTableStorage:
+		return "table_storage"
+	case ServiceWebApp:
+		return "web_app"
+	default:
+		return "unknown"
+	}
+}
+
+// Source is where a candidate was discovered.
+type Source uint8
+
+const (
+	SourceUnknown Source = iota
+	SourceDirectCT
+	SourceCNAME
+	SourceDNSAlias
+)
+
+func (s Source) String() string {
+	switch s {
+	case SourceDirectCT:
+		return "direct_ct"
+	case SourceCNAME:
+		return "cname"
+	case SourceDNSAlias:
+		return "dns_alias"
+	default:
+		return "unknown"
+	}
+}
+
+// PublicSemantic captures when the hostname format itself carries public
+// accessibility meaning. It never asserts that data is actually exposed.
+type PublicSemantic uint8
+
+const (
+	PublicSemanticNone PublicSemantic = iota
+	PublicSemanticPublicEndpoint
+	PublicSemanticVPCEndpoint
+	PublicSemanticPublicURL
+)
+
+func (p PublicSemantic) String() string {
+	switch p {
+	case PublicSemanticPublicEndpoint:
+		return "public_endpoint"
+	case PublicSemanticVPCEndpoint:
+		return "vpc_endpoint"
+	case PublicSemanticPublicURL:
+		return "public_url"
+	default:
+		return "none"
+	}
+}
+
+// ExposureState is the outcome of an optional safe reachability check.
+type ExposureState uint8
+
+const (
+	ExposureUnknown ExposureState = iota
+	ExposurePublicDNS
+	ExposureReachable
+	ExposureAuthRequired
+	ExposurePublicRead
+	ExposurePrivate
+	ExposureNotFound
+)
+
+func (e ExposureState) String() string {
+	switch e {
+	case ExposurePublicDNS:
+		return "public_dns"
+	case ExposureReachable:
+		return "reachable"
+	case ExposureAuthRequired:
+		return "auth_required"
+	case ExposurePublicRead:
+		return "public_read"
+	case ExposurePrivate:
+		return "private"
+	case ExposureNotFound:
+		return "not_found"
+	default:
+		return "unknown"
+	}
+}
+
+// Match is the compact result of classifying a single hostname. The string
+// fields are substrings of the input hostname (zero-copy).
+type Match struct {
+	Provider         Provider
+	Service          Service
+	PublicSemantic   PublicSemantic
+	ResourceSpecific bool
+	Region           string
+	Resource         string
+}
+
+// Candidate is a provider resource discovered directly or through DNS.
+type Candidate struct {
+	Hostname         string // hostname that matched (provider hostname)
+	Original         string // original CT hostname (equal to Hostname for direct matches)
+	Region           string
+	Resource         string
+	Provider         Provider
+	Service          Service
+	Source           Source
+	PublicSemantic   PublicSemantic
+	ResourceSpecific bool
+}

internal/pipeline/pipeline.go

@@ -0,0 +1,345 @@
+// Package pipeline wires the in-process CertSpotter identifier callback to the
+// classifier, DNS enrichment and the batched PostgreSQL writer using bounded
+// queues throughout.
+package pipeline
+
+import (
+	"context"
+	"net"
+	"sync"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+
+	"tea.karlcom.de/karl/bucketeer/internal/classify"
+	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+// Finding is a fully enriched candidate ready to be persisted.
+type Finding struct {
+	model.Candidate
+	CNAMEChain []string
+	IPv4       []string
+	IPv6       []string
+}
+
+// WriteFunc persists a batch of findings.
+type WriteFunc func(ctx context.Context, batch []Finding) error
+
+type Config struct {
+	DirectQueueSize int
+	DNSQueueSize    int
+	DBQueueSize     int
+	DirectWorkers   int
+	DNSWorkers      int
+	BatchSize       int
+	FlushInterval   time.Duration
+	DedupTTL        time.Duration
+	DedupSize       int
+
+	Enricher *dns.Enricher
+	Write    WriteFunc
+	Metrics  *metrics.Metrics
+}
+
+// Pipeline is the certificate-to-database data flow. It is safe to call
+// HandleIdentifiers from the CertSpotter monitor goroutines concurrently.
+type Pipeline struct {
+	cfg Config
+
+	directQ chan model.Candidate
+	dnsQ    chan string
+	dbQ     chan Finding
+
+	dedup    *dns.TTLCache[bool] // hostnames already queued for DNS
+	seen     *dns.TTLCache[bool] // provider+service+provider_hostname already written
+	producer sync.WaitGroup
+	workers  sync.WaitGroup
+}
+
+func New(cfg Config) *Pipeline {
+	if cfg.DirectQueueSize <= 0 {
+		cfg.DirectQueueSize = 8192
+	}
+	if cfg.DNSQueueSize <= 0 {
+		cfg.DNSQueueSize = 8192
+	}
+	if cfg.DBQueueSize <= 0 {
+		cfg.DBQueueSize = 8192
+	}
+	if cfg.DirectWorkers <= 0 {
+		cfg.DirectWorkers = 2
+	}
+	if cfg.DNSWorkers <= 0 {
+		cfg.DNSWorkers = 16
+	}
+	if cfg.BatchSize <= 0 {
+		cfg.BatchSize = 500
+	}
+	if cfg.FlushInterval <= 0 {
+		cfg.FlushInterval = 2 * time.Second
+	}
+	if cfg.DedupTTL <= 0 {
+		cfg.DedupTTL = time.Hour
+	}
+	if cfg.DedupSize <= 0 {
+		cfg.DedupSize = 1 << 20
+	}
+	p := &Pipeline{
+		cfg:     cfg,
+		directQ: make(chan model.Candidate, cfg.DirectQueueSize),
+		dnsQ:    make(chan string, cfg.DNSQueueSize),
+		dbQ:     make(chan Finding, cfg.DBQueueSize),
+		dedup:   dns.NewTTLCache[bool](cfg.DedupSize, 64),
+		seen:    dns.NewTTLCache[bool](cfg.DedupSize, 64),
+	}
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) })
+		p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(len(p.dnsQ)) })
+		p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) })
+	}
+	return p
+}
+
+// Start launches the worker pools. Close must be called to drain.
+func (p *Pipeline) Start(ctx context.Context) {
+	for i := 0; i < p.cfg.DirectWorkers; i++ {
+		p.producer.Add(1)
+		go p.directWorker(ctx)
+	}
+	for i := 0; i < p.cfg.DNSWorkers; i++ {
+		p.producer.Add(1)
+		go p.dnsWorker(ctx)
+	}
+	p.workers.Add(1)
+	go p.dbWorker(ctx)
+}
+
+// Close stops intake and drains all queues. It must only be called after the
+// CertSpotter monitor has stopped calling HandleIdentifiers.
+func (p *Pipeline) Close() {
+	close(p.directQ)
+	close(p.dnsQ)
+	p.producer.Wait()
+	close(p.dbQ)
+	p.workers.Wait()
+}
+
+// HandleIdentifiers implements certspotter.IdentifierSink. It is deliberately
+// allocation-light and never blocks: queues are bounded and overflow is dropped.
+func (p *Pipeline) HandleIdentifiers(ctx context.Context, ids *certspotter.Identifiers) {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.CTCertificates.Add(1)
+		p.cfg.Metrics.CTIdentifiers.Add(uint64(len(ids.DNSNames)))
+	}
+	for _, host := range ids.DNSNames {
+		p.HandleHost(host)
+	}
+}
+
+// HandleHost classifies a single hostname and enqueues it for direct handling
+// or, when it does not match, for DNS/CNAME enrichment.
+func (p *Pipeline) HandleHost(host string) {
+	if m, ok := classify.Classify(host); ok {
+		if !m.ResourceSpecific {
+			// Generic service endpoints (bucket name in the path) are not
+			// customer resources; do not create findings for them.
+			return
+		}
+		if p.cfg.Metrics != nil {
+			p.cfg.Metrics.DirectMatches.Add(1)
+			p.cfg.Metrics.Candidates.Add(1)
+		}
+		if !p.markSeen(host) {
+			return
+		}
+		cand := candidateFrom(host, host, m, model.SourceDirectCT)
+		select {
+		case p.directQ <- cand:
+		default:
+			p.drop()
+		}
+		return
+	}
+	if !classify.ShouldEnrich(host) {
+		return
+	}
+	if _, dup := p.dedup.Get(host); dup {
+		return
+	}
+	p.dedup.Set(host, true, p.cfg.DedupTTL)
+	select {
+	case p.dnsQ <- host:
+	default:
+		p.drop()
+	}
+}
+
+func (p *Pipeline) drop() {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.QueueFull.Add(1)
+		p.cfg.Metrics.QueueDrops.Add(1)
+	}
+}
+
+func candidateFrom(host, original string, m model.Match, src model.Source) model.Candidate {
+	return model.Candidate{
+		Hostname:         host,
+		Original:         original,
+		Region:           m.Region,
+		Resource:         m.Resource,
+		Provider:         m.Provider,
+		Service:          m.Service,
+		Source:           src,
+		PublicSemantic:   m.PublicSemantic,
+		ResourceSpecific: m.ResourceSpecific,
+	}
+}
+
+func appendIPs(f *Finding, ips []net.IP) {
+	for _, ip := range ips {
+		if v4 := ip.To4(); v4 != nil {
+			f.IPv4 = append(f.IPv4, v4.String())
+		} else {
+			f.IPv6 = append(f.IPv6, ip.String())
+		}
+	}
+}
+
+func (p *Pipeline) enrich(ctx context.Context, c model.Candidate) Finding {
+	f := Finding{Candidate: c}
+	if p.cfg.Enricher == nil {
+		return f
+	}
+	res, ok := p.cfg.Enricher.Resolve(ctx, c.Hostname)
+	if !ok {
+		return f
+	}
+	f.CNAMEChain = res.Chain
+	appendIPs(&f, res.IPs)
+	return f
+}
+
+func (p *Pipeline) directWorker(ctx context.Context) {
+	defer p.producer.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case c, ok := <-p.directQ:
+			if !ok {
+				return
+			}
+			p.pushDB(ctx, p.enrich(ctx, c))
+		}
+	}
+}
+
+func (p *Pipeline) dnsWorker(ctx context.Context) {
+	defer p.producer.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case host, ok := <-p.dnsQ:
+			if !ok {
+				return
+			}
+			if len(p.dbQ) >= cap(p.dbQ)/2 {
+				// Load shedding: optional CNAME enrichment is dropped before
+				// direct matches when the database writer is backed up.
+				p.drop()
+				continue
+			}
+			p.enrichHost(ctx, host)
+		}
+	}
+}
+
+func (p *Pipeline) enrichHost(ctx context.Context, host string) {
+	res, ok := p.cfg.Enricher.Resolve(ctx, host)
+	if !ok {
+		return
+	}
+	for _, target := range res.Chain[1:] {
+		m, matched := classify.Classify(target)
+		if !matched || !m.ResourceSpecific {
+			continue
+		}
+		if p.cfg.Metrics != nil {
+			p.cfg.Metrics.CNAMEMatches.Add(1)
+			p.cfg.Metrics.Candidates.Add(1)
+		}
+		if !p.markSeen(target) {
+			continue
+		}
+		f := Finding{
+			Candidate:  candidateFrom(target, host, m, model.SourceCNAME),
+			CNAMEChain: res.Chain,
+		}
+		appendIPs(&f, res.IPs)
+		p.pushDB(ctx, f)
+	}
+}
+
+// markSeen dedups by hostname alone: a given hostname always classifies to the
+// same provider/service, so no key concatenation (and no allocation) is needed.
+func (p *Pipeline) markSeen(host string) bool {
+	if _, dup := p.seen.Get(host); dup {
+		return false
+	}
+	p.seen.Set(host, true, p.cfg.DedupTTL)
+	return true
+}
+
+func (p *Pipeline) pushDB(ctx context.Context, f Finding) {
+	select {
+	case p.dbQ <- f:
+	default:
+		p.drop()
+	}
+}
+
+func (p *Pipeline) dbWorker(ctx context.Context) {
+	defer p.workers.Done()
+	batch := make([]Finding, 0, p.cfg.BatchSize)
+	ticker := time.NewTicker(p.cfg.FlushInterval)
+	defer ticker.Stop()
+
+	flush := func() {
+		if len(batch) == 0 {
+			return
+		}
+		if p.cfg.Write != nil {
+			if err := p.cfg.Write(ctx, batch); err != nil {
+				if p.cfg.Metrics != nil {
+					p.cfg.Metrics.DBErrors.Add(1)
+				}
+			} else if p.cfg.Metrics != nil {
+				p.cfg.Metrics.DBRows.Add(uint64(len(batch)))
+			}
+		}
+		batch = batch[:0]
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			flush()
+			return
+		case f, ok := <-p.dbQ:
+			if !ok {
+				flush()
+				return
+			}
+			batch = append(batch, f)
+			if len(batch) >= p.cfg.BatchSize {
+				flush()
+			}
+		case <-ticker.C:
+			flush()
+		}
+	}
+}

internal/pipeline/pipeline_test.go

@@ -0,0 +1,236 @@
+package pipeline
+
+import (
+	"context"
+	"net"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+
+	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func makeHosts(prefix, suffix string) []string {
+	hosts := make([]string, 256)
+	for i := range hosts {
+		hosts[i] = prefix + strconv.Itoa(i) + suffix
+	}
+	return hosts
+}
+
+// TestHandleHostNoAllocs audits the sink hot path: after warmup, classifying and
+// enqueueing a hostname must not allocate.
+func TestHandleHostNoAllocs(t *testing.T) {
+	p := New(Config{DNSQueueSize: 1 << 20, DedupSize: 1 << 20})
+	nonMatch := makeHosts("api", ".example.com")
+	direct := makeHosts("bucket", ".s3.eu-central-1.amazonaws.com")
+	for _, h := range nonMatch {
+		p.HandleHost(h)
+	}
+	for _, h := range direct {
+		p.HandleHost(h)
+	}
+	allocs := testing.AllocsPerRun(2000, func() {
+		for i := 0; i < 64; i++ {
+			p.HandleHost(nonMatch[i])
+			p.HandleHost(direct[i])
+		}
+	})
+	if allocs != 0 {
+		t.Fatalf("HandleHost allocates %.3f allocs/op, want 0", allocs)
+	}
+}
+
+func BenchmarkHandleHostNonMatch(b *testing.B) {
+	p := New(Config{DNSQueueSize: 1 << 20, DedupSize: 1 << 20})
+	hosts := makeHosts("api", ".example.com")
+	for _, h := range hosts {
+		p.HandleHost(h)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.HandleHost(hosts[i&255])
+	}
+}
+
+func BenchmarkHandleHostDirectMatch(b *testing.B) {
+	p := New(Config{DirectQueueSize: 1 << 20, DedupSize: 1 << 20})
+	hosts := makeHosts("bucket", ".s3.eu-central-1.amazonaws.com")
+	for _, h := range hosts {
+		p.HandleHost(h)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.HandleHost(hosts[i&255])
+	}
+}
+
+type fakeResolver struct {
+	cnames map[string]string
+	ips    map[string][]net.IP
+}
+
+func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
+	if c, ok := f.cnames[host]; ok {
+		return c, nil
+	}
+	return host, nil
+}
+
+func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
+	return f.ips[host], nil
+}
+
+func newEnricher(r dns.Resolver) *dns.Enricher {
+	return &dns.Enricher{
+		Resolver:    r,
+		Cache:       dns.NewTTLCache[dns.Result](1024, 8),
+		PositiveTTL: time.Minute,
+		NegativeTTL: time.Minute,
+		MaxDepth:    8,
+	}
+}
+
+type collector struct {
+	mu    sync.Mutex
+	items []Finding
+}
+
+func (c *collector) write(ctx context.Context, batch []Finding) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.items = append(c.items, batch...)
+	return nil
+}
+
+func (c *collector) all() []Finding {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return append([]Finding(nil), c.items...)
+}
+
+func TestDirectMatch(t *testing.T) {
+	col := &collector{}
+	m := metrics.New()
+	p := New(Config{
+		Enricher:  newEnricher(&fakeResolver{}),
+		Write:     col.write,
+		Metrics:   m,
+		BatchSize: 10,
+	})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{"bucket.s3.eu-central-1.amazonaws.com"}})
+	time.Sleep(50 * time.Millisecond)
+	p.Close()
+
+	items := col.all()
+	if len(items) != 1 {
+		t.Fatalf("got %d findings, want 1 (%+v)", len(items), items)
+	}
+	f := items[0]
+	if f.Provider != model.ProviderAWS || f.Service != model.ServiceS3 || f.Source != model.SourceDirectCT {
+		t.Fatalf("unexpected finding %+v", f)
+	}
+	if m.DirectMatches.Load() != 1 {
+		t.Fatalf("direct matches = %d, want 1", m.DirectMatches.Load())
+	}
+}
+
+func TestCNAMEMatch(t *testing.T) {
+	col := &collector{}
+	r := &fakeResolver{
+		cnames: map[string]string{"files.example.com": "files.s3.amazonaws.com"},
+		ips:    map[string][]net.IP{"files.example.com": {net.ParseIP("1.2.3.4")}},
+	}
+	m := metrics.New()
+	p := New(Config{Enricher: newEnricher(r), Write: col.write, Metrics: m, BatchSize: 10})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{"files.example.com"}})
+	time.Sleep(50 * time.Millisecond)
+	p.Close()
+
+	items := col.all()
+	if len(items) != 1 {
+		t.Fatalf("got %d findings, want 1", len(items))
+	}
+	f := items[0]
+	if f.Source != model.SourceCNAME || f.Hostname != "files.s3.amazonaws.com" || f.Original != "files.example.com" {
+		t.Fatalf("unexpected finding %+v", f)
+	}
+	if len(f.IPv4) != 1 || f.IPv4[0] != "1.2.3.4" {
+		t.Fatalf("unexpected IPs %+v", f.IPv4)
+	}
+	if m.CNAMEMatches.Load() != 1 {
+		t.Fatalf("cname matches = %d, want 1", m.CNAMEMatches.Load())
+	}
+}
+
+func TestCNAMEMatchIPv6(t *testing.T) {
+	col := &collector{}
+	r := &fakeResolver{
+		cnames: map[string]string{"db.example.com": "mydb.cluster-abc.us-east-1.rds.amazonaws.com"},
+		ips:    map[string][]net.IP{"db.example.com": {net.ParseIP("2606:4700::1111")}},
+	}
+	p := New(Config{Enricher: newEnricher(r), Write: col.write, BatchSize: 10})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleHost("db.example.com")
+	time.Sleep(50 * time.Millisecond)
+	p.Close()
+	items := col.all()
+	if len(items) != 1 || len(items[0].IPv6) != 1 || items[0].IPv6[0] != "2606:4700::1111" {
+		t.Fatalf("unexpected IPv6 finding: %+v", items)
+	}
+}
+
+func TestQueueSaturationDrops(t *testing.T) {
+	m := metrics.New()
+	p := New(Config{DirectQueueSize: 1, DNSQueueSize: 1, Metrics: m})
+	// Intentionally do not Start: the queue stays full.
+	p.HandleHost("bucket.s3.amazonaws.com")
+	p.HandleHost("other.s3.amazonaws.com")
+	if m.QueueDrops.Load() == 0 {
+		t.Fatal("expected a drop when the direct queue is full")
+	}
+	if m.Candidates.Load() != 2 {
+		t.Fatalf("candidates = %d, want 2", m.Candidates.Load())
+	}
+}
+
+func TestDNSDedup(t *testing.T) {
+	m := metrics.New()
+	p := New(Config{DNSQueueSize: 8, Metrics: m})
+	p.HandleHost("files.example.com")
+	p.HandleHost("files.example.com")
+	if got := len(p.dnsQ); got != 1 {
+		t.Fatalf("dns queue len = %d, want 1", got)
+	}
+}
+
+func TestGenericEndpointNotStored(t *testing.T) {
+	col := &collector{}
+	p := New(Config{Write: col.write, BatchSize: 10})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Start(ctx)
+	p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{
+		"objectstorage.us-ashburn-1.oraclecloud.com",
+	}})
+	time.Sleep(30 * time.Millisecond)
+	p.Close()
+	if items := col.all(); len(items) != 0 {
+		t.Fatalf("generic endpoint produced findings: %+v", items)
+	}
+}

internal/probe/probe.go

@@ -0,0 +1,142 @@
+// Package probe performs optional, minimal, non-destructive reachability checks.
+// It never authenticates, downloads objects, mutates anything, or runs database
+// queries; it only observes the response to a single HTTPS GET /.
+package probe
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+// UpdateFunc records the observed exposure state.
+type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState)
+
+type Config struct {
+	Workers int
+	Timeout time.Duration
+	Queue   int
+	Metrics *metrics.Metrics
+	Update  UpdateFunc
+}
+
+// Prober runs a fixed pool of workers over a bounded queue.
+type Prober struct {
+	cfg    Config
+	queue  chan model.Candidate
+	client *http.Client
+	wg     sync.WaitGroup
+}
+
+func New(cfg Config) *Prober {
+	if cfg.Workers <= 0 {
+		cfg.Workers = 4
+	}
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = 5 * time.Second
+	}
+	if cfg.Queue <= 0 {
+		cfg.Queue = 1024
+	}
+	return &Prober{
+		cfg:   cfg,
+		queue: make(chan model.Candidate, cfg.Queue),
+		client: &http.Client{
+			Timeout: cfg.Timeout,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+			Transport: &http.Transport{
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true}, // observation only, no trust decision
+				MaxIdleConns:        cfg.Workers * 2,
+				MaxIdleConnsPerHost: 2,
+				IdleConnTimeout:     30 * time.Second,
+				DialContext: (&net.Dialer{
+					Timeout:   cfg.Timeout,
+					KeepAlive: 30 * time.Second,
+				}).DialContext,
+			},
+		},
+	}
+}
+
+func (p *Prober) Start(ctx context.Context) {
+	for i := 0; i < p.cfg.Workers; i++ {
+		p.wg.Add(1)
+		go p.worker(ctx)
+	}
+}
+
+// Submit enqueues a candidate without blocking; it drops when the queue is full.
+func (p *Prober) Submit(c model.Candidate) {
+	if p.cfg.Metrics != nil {
+		p.cfg.Metrics.ProbeAttempts.Add(1)
+	}
+	select {
+	case p.queue <- c:
+	default:
+	}
+}
+
+func (p *Prober) Wait() { p.wg.Wait() }
+
+func (p *Prober) worker(ctx context.Context) {
+	defer p.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case c, ok := <-p.queue:
+			if !ok {
+				return
+			}
+			state := p.probe(ctx, c)
+			if p.cfg.Metrics != nil && state == model.ExposurePublicRead {
+				p.cfg.Metrics.ProbePublic.Add(1)
+			}
+			if p.cfg.Update != nil {
+				p.cfg.Update(ctx, c, state)
+			}
+		}
+	}
+}
+
+func (p *Prober) probe(ctx context.Context, c model.Candidate) model.ExposureState {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+c.Hostname+"/", nil)
+	if err != nil {
+		return model.ExposureUnknown
+	}
+	resp, err := p.client.Do(req)
+	if err != nil {
+		var dnsErr *net.DNSError
+		if errors.As(err, &dnsErr) {
+			return model.ExposurePrivate
+		}
+		return model.ExposureUnknown
+	}
+	defer resp.Body.Close()
+
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		return model.ExposurePublicRead
+	case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized:
+		return model.ExposureAuthRequired
+	case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone:
+		return model.ExposureNotFound
+	default:
+		return model.ExposureReachable
+	}
+}
+
+// Describe is a helper for logs/tests.
+func Describe(s model.ExposureState) string {
+	return fmt.Sprintf("exposure=%s", s)
+}

internal/probe/probe_test.go

@@ -0,0 +1,45 @@
+package probe
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+)
+
+func newProber(t *testing.T, status int) (*Prober, string) {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(status)
+	}))
+	t.Cleanup(srv.Close)
+	p := New(Config{Workers: 1, Timeout: time.Second})
+	// httptest uses a self-signed cert; the prober already skips verification.
+	_ = tls.Config{}
+	host := strings.TrimPrefix(srv.URL, "https://")
+	return p, host
+}
+
+func TestProbeClassification(t *testing.T) {
+	cases := []struct {
+		status int
+		want   model.ExposureState
+	}{
+		{http.StatusOK, model.ExposurePublicRead},
+		{http.StatusForbidden, model.ExposureAuthRequired},
+		{http.StatusUnauthorized, model.ExposureAuthRequired},
+		{http.StatusNotFound, model.ExposureNotFound},
+		{http.StatusInternalServerError, model.ExposureReachable},
+	}
+	for _, c := range cases {
+		p, host := newProber(t, c.status)
+		got := p.probe(t.Context(), model.Candidate{Hostname: host})
+		if got != c.want {
+			t.Errorf("status %d: got %s, want %s", c.status, got, c.want)
+		}
+	}
+}

internal/store/state.go

@@ -0,0 +1,165 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"strconv"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/monitor"
+)
+
+var _ monitor.StateProvider = (*Store)(nil)
+
+// Prepare implements monitor.StateProvider. It applies the embedded schema so
+// the container needs no writable filesystem.
+func (s *Store) Prepare(ctx context.Context) error {
+	if err := s.Migrate(ctx); err != nil {
+		return err
+	}
+	return nil
+}
+
+// PrepareLog is a no-op: checkpoint rows are created lazily by StoreLogState.
+func (s *Store) PrepareLog(ctx context.Context, logID monitor.LogID) error { return nil }
+
+// StoreLogState persists the certspotter monitor position for a log.
+func (s *Store) StoreLogState(ctx context.Context, logID monitor.LogID, state *monitor.LogState) error {
+	data, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+	_, err = s.pool.Exec(ctx, `
+		INSERT INTO ct_checkpoints (log_id, state, updated_at)
+		VALUES ($1, $2, now())
+		ON CONFLICT (log_id) DO UPDATE SET state = EXCLUDED.state, updated_at = now()`,
+		logID.Base64URLString(), data)
+	return err
+}
+
+// LoadLogState returns nil when no checkpoint exists for the log.
+func (s *Store) LoadLogState(ctx context.Context, logID monitor.LogID) (*monitor.LogState, error) {
+	var data []byte
+	err := s.pool.QueryRow(ctx, `SELECT state FROM ct_checkpoints WHERE log_id = $1`, logID.Base64URLString()).Scan(&data)
+	if err == pgx.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	state := new(monitor.LogState)
+	if err := json.Unmarshal(data, state); err != nil {
+		return nil, fmt.Errorf("corrupt checkpoint for log %s: %w", logID.Base64URLString(), err)
+	}
+	return state, nil
+}
+
+func sthKey(sth *cttypes.SignedTreeHead) string {
+	return strconv.FormatUint(sth.TreeSize, 10) + "-" +
+		strconv.FormatUint(sth.Timestamp, 10) + "-" +
+		sth.RootHash.Base64String()
+}
+
+// StoreSTH stores an unverified signed tree head. If it already exists the
+// original stored_at is preserved.
+func (s *Store) StoreSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) (*monitor.StoredSTH, error) {
+	data, err := json.Marshal(sth)
+	if err != nil {
+		return nil, err
+	}
+	var storedAt time.Time
+	err = s.pool.QueryRow(ctx, `
+		INSERT INTO ct_sths (log_id, sth_key, tree_size, sth, stored_at)
+		VALUES ($1, $2, $3, $4, now())
+		ON CONFLICT (log_id, sth_key) DO UPDATE SET log_id = EXCLUDED.log_id
+		RETURNING stored_at`,
+		logID.Base64URLString(), sthKey(sth), sth.TreeSize, data).Scan(&storedAt)
+	if err != nil {
+		return nil, err
+	}
+	return &monitor.StoredSTH{SignedTreeHead: *sth, StoredAt: storedAt}, nil
+}
+
+// LoadSTHs loads all stored STHs for a log, sorted by tree size.
+func (s *Store) LoadSTHs(ctx context.Context, logID monitor.LogID) ([]*monitor.StoredSTH, error) {
+	rows, err := s.pool.Query(ctx,
+		`SELECT sth, stored_at FROM ct_sths WHERE log_id = $1 ORDER BY tree_size ASC`,
+		logID.Base64URLString())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []*monitor.StoredSTH
+	for rows.Next() {
+		var (
+			data     []byte
+			storedAt time.Time
+		)
+		if err := rows.Scan(&data, &storedAt); err != nil {
+			return nil, err
+		}
+		var sth cttypes.SignedTreeHead
+		if err := json.Unmarshal(data, &sth); err != nil {
+			return nil, err
+		}
+		out = append(out, &monitor.StoredSTH{SignedTreeHead: sth, StoredAt: storedAt})
+	}
+	return out, rows.Err()
+}
+
+// RemoveSTH deletes an STH once it has been verified against the Merkle tree.
+func (s *Store) RemoveSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) error {
+	_, err := s.pool.Exec(ctx, `DELETE FROM ct_sths WHERE log_id = $1 AND sth_key = $2`,
+		logID.Base64URLString(), sthKey(sth))
+	return err
+}
+
+// StoreIssuer and LoadIssuer are no-ops. ctcloud's identifier sink bypasses
+// certificate-chain retrieval entirely, so the issuer cache is never consulted;
+// keeping it empty also satisfies the "no certificate contents in the state
+// store" requirement. The Go runtime cache handles any incidental calls.
+func (s *Store) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error {
+	return nil
+}
+
+func (s *Store) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	return nil, nil
+}
+
+// NotifyCert is never called because the identifier sink handles certificates.
+func (s *Store) NotifyCert(ctx context.Context, cert *monitor.DiscoveredCert) error { return nil }
+
+// NotifyMalformedEntry counts unparseable log entries without logging per entry.
+func (s *Store) NotifyMalformedEntry(ctx context.Context, entry *monitor.LogEntry, parseError error) error {
+	if s.metrics != nil {
+		s.metrics.MalformedCT.Add(1)
+	}
+	return nil
+}
+
+// NotifyHealthCheckFailure is rare (once per health-check interval at most), so
+// it is safe to log.
+func (s *Store) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info monitor.HealthCheckFailure) error {
+	log.Printf("ct health check failure: %s", info.Summary())
+	return nil
+}
+
+// NotifyError counts transient CT errors. It deliberately does not log, to keep
+// logging off the high-frequency error path.
+func (s *Store) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error {
+	if s.metrics != nil {
+		s.metrics.CTErrors.Add(1)
+	}
+	return nil
+}
+
+// GetErrors returns the recent CT errors. ctcloud keeps only aggregate counters.
+func (s *Store) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
+	return "", nil
+}

internal/store/store.go

@@ -0,0 +1,146 @@
+// Package store owns the PostgreSQL schema, the batched findings writer and the
+// restart-safe CT checkpoint state provider.
+package store
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"tea.karlcom.de/karl/bucketeer/internal/metrics"
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
+	"tea.karlcom.de/karl/bucketeer/migrations"
+)
+
+// Store wraps a pgx pool with the operations ctcloud needs.
+type Store struct {
+	pool    *pgxpool.Pool
+	metrics *metrics.Metrics
+}
+
+// Open connects to PostgreSQL. The pool is bounded; the caller configures the
+// maximum number of connections.
+func Open(ctx context.Context, databaseURL string, maxConns int) (*Store, error) {
+	cfg, err := pgxpool.ParseConfig(databaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid DATABASE_URL: %w", err)
+	}
+	if maxConns > 0 {
+		cfg.MaxConns = int32(maxConns)
+	}
+	cfg.MaxConnLifetime = 30 * time.Minute
+	cfg.MinConns = 1
+	pool, err := pgxpool.NewWithConfig(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &Store{pool: pool}, nil
+}
+
+func (s *Store) SetMetrics(m *metrics.Metrics) { s.metrics = m }
+
+func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
+
+func (s *Store) Close() { s.pool.Close() }
+
+func (s *Store) Pool() *pgxpool.Pool { return s.pool }
+
+// Migrate applies the embedded schema idempotently.
+func (s *Store) Migrate(ctx context.Context) error {
+	for _, name := range migrations.SQL() {
+		sqlBytes, err := migrations.FS.ReadFile(name)
+		if err != nil {
+			return err
+		}
+		if _, err := s.pool.Exec(ctx, string(sqlBytes)); err != nil {
+			return fmt.Errorf("applying %s: %w", name, err)
+		}
+	}
+	return nil
+}
+
+const upsertFindingSQL = `
+WITH asset AS (
+    INSERT INTO cloud_assets
+        (provider, service, hostname, provider_hostname, resource_identifier,
+         region, resource_specific, public_semantic, first_seen, last_seen)
+    VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now())
+    ON CONFLICT (provider, service, provider_hostname)
+    DO UPDATE SET
+        last_seen           = now(),
+        hostname            = EXCLUDED.hostname,
+        resource_identifier = CASE WHEN EXCLUDED.resource_identifier = '' THEN cloud_assets.resource_identifier ELSE EXCLUDED.resource_identifier END,
+        region              = CASE WHEN EXCLUDED.region = '' THEN cloud_assets.region ELSE EXCLUDED.region END
+    RETURNING id
+)
+INSERT INTO observations
+    (asset_id, source_hostname, source_type, cname_chain, ipv4, ipv6)
+SELECT asset.id, $9, $10, $11, $12, $13 FROM asset`
+
+// WriteFindings writes one batch of findings in a single round trip. Asset rows
+// are upserted and an observation is appended for each finding.
+func (s *Store) WriteFindings(ctx context.Context, batch []pipeline.Finding) error {
+	if len(batch) == 0 {
+		return nil
+	}
+	var lastErr error
+	// ponytail: three quick attempts; the pipeline keeps running and the
+	// bounded DB queue sheds load if PostgreSQL stays down.
+	for attempt := 0; attempt < 3; attempt++ {
+		if err := s.writeBatch(ctx, batch); err != nil {
+			lastErr = err
+			if ctx.Err() != nil {
+				return err
+			}
+			backoff := time.Duration(1<<attempt) * 100 * time.Millisecond
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(backoff):
+			}
+			continue
+		}
+		return nil
+	}
+	return lastErr
+}
+
+func (s *Store) writeBatch(ctx context.Context, batch []pipeline.Finding) error {
+	b := &pgx.Batch{}
+	for i := range batch {
+		f := &batch[i]
+		b.Queue(upsertFindingSQL,
+			f.Provider.String(), f.Service.String(), f.Original, f.Hostname,
+			f.Resource, f.Region, f.ResourceSpecific, f.PublicSemantic.String(),
+			f.Original, f.Source.String(),
+			nonNil(f.CNAMEChain), nonNil(f.IPv4), nonNil(f.IPv6),
+		)
+	}
+	br := s.pool.SendBatch(ctx, b)
+	defer br.Close()
+	for range batch {
+		if _, err := br.Exec(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func nonNil(s []string) []string {
+	if s == nil {
+		return []string{}
+	}
+	return s
+}
+
+// UpdateExposure records the outcome of an optional reachability probe.
+func (s *Store) UpdateExposure(ctx context.Context, provider, service, providerHostname string, state model.ExposureState) error {
+	_, err := s.pool.Exec(ctx,
+		`UPDATE cloud_assets SET exposure_state = $4 WHERE provider = $1 AND service = $2 AND provider_hostname = $3`,
+		provider, service, providerHostname, state.String())
+	return err
+}

internal/store/store_test.go

@@ -0,0 +1,66 @@
+package store
+
+import (
+	"os"
+	"testing"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+
+	"tea.karlcom.de/karl/bucketeer/internal/model"
+	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
+)
+
+func TestSTHKeyStableAndUnique(t *testing.T) {
+	a := &cttypes.SignedTreeHead{TreeSize: 10, Timestamp: 100}
+	if sthKey(a) != sthKey(a) {
+		t.Fatal("sthKey is not deterministic")
+	}
+	b := &cttypes.SignedTreeHead{TreeSize: 11, Timestamp: 100}
+	if sthKey(a) == sthKey(b) {
+		t.Fatal("sthKey collides for different tree sizes")
+	}
+}
+
+func TestNonNil(t *testing.T) {
+	if got := nonNil(nil); got == nil || len(got) != 0 {
+		t.Fatalf("nonNil(nil) = %#v", got)
+	}
+	in := []string{"a"}
+	if got := nonNil(in); len(got) != 1 || got[0] != "a" {
+		t.Fatalf("nonNil changed slice: %#v", got)
+	}
+}
+
+// TestIntegration exercises the real schema when DATABASE_URL is set.
+func TestIntegration(t *testing.T) {
+	url := os.Getenv("DATABASE_URL")
+	if url == "" {
+		t.Skip("DATABASE_URL not set; skipping integration test")
+	}
+	ctx := t.Context()
+	s, err := Open(ctx, url, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	if err := s.Migrate(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	f := pipeline.Finding{
+		Candidate: model.Candidate{
+			Hostname: "it-bucket.s3.eu-central-1.amazonaws.com",
+			Original: "it-bucket.s3.eu-central-1.amazonaws.com",
+			Provider: model.ProviderAWS,
+			Service:  model.ServiceS3,
+			Source:   model.SourceDirectCT,
+		},
+		IPv4: []string{"1.2.3.4"},
+	}
+	if err := s.WriteFindings(ctx, []pipeline.Finding{f}); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.UpdateExposure(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, model.ExposureReachable); err != nil {
+		t.Fatal(err)
+	}
+}

migrations/0001_schema.sql

@@ -0,0 +1,56 @@
+-- ctcloud schema. Applied idempotently at startup; there is no external
+-- migration tooling so the container needs no writable filesystem.
+
+CREATE TABLE IF NOT EXISTS cloud_assets (
+    id                  BIGSERIAL PRIMARY KEY,
+    provider            TEXT        NOT NULL,
+    service             TEXT        NOT NULL,
+    hostname            TEXT        NOT NULL, -- original CT hostname (or CNAME source)
+    provider_hostname   TEXT        NOT NULL, -- hostname that matched the provider
+    resource_identifier TEXT        NOT NULL DEFAULT '',
+    region              TEXT        NOT NULL DEFAULT '',
+    resource_specific   BOOLEAN     NOT NULL DEFAULT TRUE,
+    public_semantic     TEXT        NOT NULL DEFAULT 'none',
+    exposure_state      TEXT        NOT NULL DEFAULT 'unknown',
+    first_seen          TIMESTAMPTZ NOT NULL DEFAULT now(),
+    last_seen           TIMESTAMPTZ NOT NULL DEFAULT now(),
+    UNIQUE (provider, service, provider_hostname)
+);
+
+CREATE INDEX IF NOT EXISTS cloud_assets_last_seen_idx ON cloud_assets (last_seen);
+
+CREATE TABLE IF NOT EXISTS observations (
+    id              BIGSERIAL PRIMARY KEY,
+    asset_id        BIGINT      NOT NULL REFERENCES cloud_assets (id) ON DELETE CASCADE,
+    observed_at     TIMESTAMPTZ NOT NULL DEFAULT now(),
+    source_hostname TEXT        NOT NULL,
+    source_type     TEXT        NOT NULL,
+    cname_chain     TEXT[]      NOT NULL DEFAULT '{}',
+    ipv4            TEXT[]      NOT NULL DEFAULT '{}',
+    ipv6            TEXT[]      NOT NULL DEFAULT '{}',
+    cert_fingerprint TEXT       NOT NULL DEFAULT '',
+    ct_log_id       TEXT        NOT NULL DEFAULT ''
+);
+
+CREATE INDEX IF NOT EXISTS observations_asset_idx ON observations (asset_id, observed_at DESC);
+
+-- CT resume state. `state` is the JSON-encoded certspotter monitor.LogState
+-- (download/verified Merkle positions and last verified STH). No certificate
+-- contents are ever stored here.
+CREATE TABLE IF NOT EXISTS ct_checkpoints (
+    log_id     TEXT        PRIMARY KEY,
+    state      JSONB       NOT NULL,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Signed tree heads downloaded but not yet verified, needed to resume safely.
+-- The full STH is kept as JSONB so the certspotter structure round-trips
+-- exactly; tree_size is duplicated as a typed column for cheap ordering.
+CREATE TABLE IF NOT EXISTS ct_sths (
+    log_id    TEXT        NOT NULL,
+    sth_key   TEXT        NOT NULL,
+    tree_size BIGINT      NOT NULL,
+    sth       JSONB       NOT NULL,
+    stored_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (log_id, sth_key)
+);

migrations/embed.go

@@ -0,0 +1,21 @@
+// Package migrations embeds the SQL schema so no runtime filesystem access is
+// required.
+package migrations
+
+import "embed"
+
+//go:embed *.sql
+var FS embed.FS
+
+// SQL returns the migration files in lexical order.
+func SQL() []string {
+	entries, err := FS.ReadDir(".")
+	if err != nil {
+		panic(err)
+	}
+	names := make([]string, 0, len(entries))
+	for _, e := range entries {
+		names = append(names, e.Name())
+	}
+	return names
+}

third_party/certspotter/.github/workflows/test.yml

@@ -0,0 +1,35 @@
+name: Test and lint Go Code
+
+on:
+  push:
+  schedule:
+    - cron: '42 9 * * *' # Runs daily at 09:42 UTC
+  workflow_dispatch:     # Allows manual triggering
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Run tests
+        run: CGO_ENABLED=1 go test -race ./...
+
+      - name: Install staticcheck
+        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+
+      - name: Run staticcheck
+        run: staticcheck ./...

third_party/certspotter/CHANGELOG.md

@@ -0,0 +1,267 @@
+# Change Log
+
+## v0.25.0 (2026-09-13)
+- Improve performance monitoring slow RFC 6962 logs whose get-entries
+  responses are interrupted.
+- Add experimental certspotter-metrics command for exporting Prometheus
+  metrics about certspotter's operation. Please leave feedback in
+  <https://github.com/SSLMate/certspotter/issues/79>.
+- Minor bug fixes and robustness improvements.
+
+## v0.24.2 (2026-06-25)
+- Extract all Common Names from Subject instead of first one in each RDN. Note
+  that modern certificate validators, including those used by all CT-enforcing
+  clients, ignore the Common Name. Therefore, any certificate missed by
+  certspotter due to this bug would not work in CT-enforcing clients anyways.
+- Support -verbose with -version to print full build info.
+- Fix some possible crashes in pathological situations.
+
+## v0.24.1 (2026-05-23)
+- Update dependencies.
+- Minor bug fixes.
+
+## v0.24.0 (2026-01-12)
+- Add certspotter-authorize command, which lets you mark legitimate certificates
+  as authorized so you don't get notified about them.
+- Deduplicate certificates and their corresponding precertificates, so you
+  receive only one notification when both are logged in CT.
+- When `-no_save` is used, write certificates to a temporary location so scripts
+  can access them via `$CERT_FILENAME`, `$JSON_FILENAME`, and `$TEXT_FILENAME`.
+
+## v0.23.0 (2025-12-05)
+- Avoid sending queries to logs in excess of the log's rate limit, if known.
+  This should improve monitoring of the TrustAsia logs.
+
+## v0.22.0 (2025-09-24)
+- Retry failed requests more aggressively so certspotter doesn't fall behind
+  downloading entries.
+- Support `$CERTSPOTTER_CACHE_DIR` for overriding cache directory location.
+- Support `$STATE_DIRECTORY`, `$CONFIGURATION_DIRECTORY`, and `$CACHE_DIRECTORY`
+  when running under systemd.
+- Truncate long error responses from logs, to keep size of error files down.
+
+## v0.21.0 (2025-07-02)
+- Instead of writing log errors to stderr, save the last 7 days worth in the
+  state directory, and include recent errors in failed health check notifications.
+- Send a meaningful User-Agent string with HTTP requests.
+
+## v0.20.1 (2025-06-19)
+- Add resilience against sendmail hanging indefinitely.
+- Add resilience against hooks which fork and keep stderr open.
+- Upgrade dependencies to latest versions.
+- Minor improvements to error handling, code quality, and efficiency.
+
+## v0.20.0 (2025-06-13)
+- Remove -batch_size option, which is obsolete due to new parallel download system.
+- Only print log errors to stderr if -verbose is specified.
+- Fix bug that could cause unverified STHs to be deleted prematurely.
+- Fail health check if log has never been successfully contacted.
+- Improve -verbose.
+- Improve documentation.
+
+## v0.19.1 (2025-05-07)
+- Fix panic when retrying failed log requests.
+- Properly log failed log requests.
+
+## v0.19.0 (2025-05-07) (RETRACTED)
+- Add support for static-ct-api logs, the next generation of CT logs.
+- Add support for downloading entries in parallel, to avoid backlogs when
+  monitoring fast-growing logs.
+- Use $EMAIL environment variable to determine sender of emails.
+- Remove submitct command.
+- Make certspotter more resilient to system crashes such as power failures.
+
+## v0.18.0 (2023-11-13)
+- Fix bug with downloading entries that did not materialize in practice
+  with any of the current logs.
+- Include `Message-ID` and `Date` in outbound emails.
+
+## v0.17.0 (2023-10-26)
+- Allow sendmail path to be configured with `$SENDMAIL_PATH`.
+- Minor improvements to documentation, efficiency.
+
+## v0.16.0 (2023-02-21)
+- Write malformed certs and failed healthchecks to filesystem so scripts
+  can access them.
+- Automatically execute scripts under `$CERTSPOTTER_CONFIG_DIR/hooks.d`
+  if it exists.
+- Automatically email addresses listed in `$CERTSPOTTER_CONFIG_DIR/email_recipients`
+  if it exists.
+
+## v0.15.1 (2023-02-09)
+- Fix some typos in help and error messages.
+- Allow version to be set via linker flag, to facilitate distro package building.
+
+## v0.15.0 (2023-02-08)
+- **Significant behavior change**: certspotter is now intended to run as
+  a daemon instead of a cron job.  Specifically, certspotter no longer
+  terminates unless it receives SIGTERM or SIGINT or there is a serious error.
+  You should remove certspotter from your crontab and arrange to run it as a
+  daemon, passing either the `-email` option or `-script` option to configure
+  how you want to be notified about certificates.
+
+  Reason for this change: although using cron made sense in the early days of
+  Certificate Transparency, certspotter now needs to run continuously to reliably
+  keep up with the high growth rate of contemporary CT logs, and to gracefully
+  handle the many transient errors that can arise when monitoring CT.
+  See <https://github.com/SSLMate/certspotter/issues/63> for background.
+
+- `-script` is now officially supported and can be used to execute
+  a command when a certificate is discovered or there is an error.  For details,
+  see the [certspotter-script(8) man page](man/certspotter-script.md).
+
+  Note the following changes from the experimental, undocumented `-script`
+  option found in previous versions:
+  - The script is also executed when there is an error.  Consult the `$EVENT`
+    variable to determine why the script was executed.
+  - The `$DNS_NAMES` and `$IP_ADDRESSES` variables have been removed because
+    the OS limits the size of environment variables and some certificates have
+    too many identifiers.  To determine a certificate's identifiers, you can
+    read the JSON file specified by the `$JSON_FILENAME` variable, as explained
+    in the [certspotter-script(8) man page](man/certspotter-script.md).
+  - The `$CERT_TYPE` variable has been removed because it is almost always
+    a serious mistake (that can make you miss malicious certificates) to treat
+    certificates and precertificates differently.  If you are currently
+    using this variable to skip precertificates, stop doing that because
+    precertificates imply the existence of a corresponding certificate that you
+    **might not** be separately notified about.  For more details, see
+    <https://github.com/SSLMate/certspotter/commit/cd2bb429fc2f4060a33ec8eb8b71a3eb12e9ba93>.
+  - New variable `$WATCH_ITEM` contains the first watch list item which
+    matched the certificate.
+
+- New `-email` option can be used to send an email when a certificate is
+  discovered or there is an error.  Your system must have a working `sendmail`
+  command.
+
+- (Behavior change) You must specify the `-stdout` option if you want discovered
+  certificates to be written to stdout.  This only makes sense when running
+  certspotter from the terminal; when running as a daemon you probably want to
+  use `-email` or `-script` instead.
+
+- Once a day, certspotter will send you a notification (per `-email` or
+  `-script`) if any problems are preventing it from detecting all certificates.
+  As in previous versions of certspotter, errors are written to stderr when they
+  occur, but since most errors are transient, you can now ignore stderr and rely
+  on the daily health check to notify you about any persistent problems that
+  require your attention.
+
+- certspotter now saves `.json` and `.txt` files alongside the `.pem` files
+  containing parsed representations of the certificate.
+
+- `.pem` files no longer have `.cert` or `.precert` in the filename.
+
+- certspotter will save its state periodically, and before terminating due to
+  SIGTERM or SIGINT, meaning it can resume monitoring without having to
+  re-download entries it has already processed.
+
+- The experimental "BygoneSSL" feature has been removed due to limited utility.
+
+- The `-num_workers` option has been removed.
+
+- The `-all_time` option has been removed. You can remove the certspotter state
+  directory if you want to re-download all entries.
+
+- The minimum supported Go version is now 1.19.
+
+## v0.14.0 (2022-06-13)
+- Switch to Go module versioning conventions.
+
+## v0.13 (2022-06-13)
+- Reduce minimum Go version to 1.17.
+- Update install instructions.
+
+## v0.12 (2022-06-07)
+- Retry failed log requests.  This should make certspotter resilient
+  to rate limiting by logs.
+- Add `-version` flag.
+- Eliminate unnecessary dependency. certspotter now depends only on
+  golang.org/x packages.
+- Switch to Go modules.
+
+## v0.11 (2021-08-17)
+- Add support for contacting logs via HTTP proxies;
+  just set the appropriate environment variable as documented at
+  <https://golang.org/pkg/net/http/#ProxyFromEnvironment>.
+- Work around RFC 6962 ambiguity related to consistency proofs
+  for empty trees.
+
+## v0.10 (2020-04-29)
+- Improve speed by processing logs in parallel
+- Add `-start_at_end` option to begin monitoring new logs at the end,
+  which significantly speeds up Cert Spotter, at the cost of missing
+  certificates that were added to a log before Cert Spotter starts
+  monitoring it
+- (Behavior change) Scan logs in their entirety the first time Cert
+  Spotter is run, unless `-start_at_end` specified (behavior change)
+- The log list is now retrieved from certspotter.org at startup instead
+  of being embedded in the source. This will allow Cert Spotter to react
+  more quickly to the frequent changes in logs.
+- (Behavior change) the `-logs` option now expects a JSON file in the v2
+  log list format. See <https://www.certificate-transparency.org/known-logs>
+  and <https://www.gstatic.com/ct/log_list/v2/log_list_schema.json>.
+- `-logs` now accepts an HTTPS URL in addition to a file path.
+- (Behavior change) the `-underwater` option has been removed. If you want
+  its behavior, specify `https://loglist.certspotter.org/underwater.json` to
+  the `-logs` option.
+
+## v0.9 (2018-04-19)
+- Add Cloudflare Nimbus logs
+- Remove Google Argon 2017 log
+- Remove WoSign and StartCom logs due to disqualification by Chromium
+  and extended downtime
+
+## v0.8 (2017-12-08)
+- Add Symantec Sirius log
+- Add DigiCert 2 log
+
+## v0.7 (2017-11-13)
+- Add Google Argon logs
+- Fix bug that caused crash on 32 bit architectures
+
+## v0.6 (2017-10-19)
+- Add Comodo Mammoth and Comodo Sabre logs
+- Minor bug fixes and improvements
+
+## v0.5 (2017-05-18)
+- Remove PuChuangSiDa 1 log due to excessive downtime and presumptive
+  disqualification from Chrome
+- Add Venafi Gen2 log
+- Improve monitoring robustness under certain pathological behavior
+  by logs
+- Minor documentation improvements
+
+## v0.4 (2017-04-03)
+- Add PuChuangSiDa 1 log
+- Remove Venafi log due to fork and disqualification from Chrome
+
+## v0.3 (2017-02-20)
+- Revise `-all_time` flag (behavior change):
+  - If `-all_time` is specified, scan the entirety of all logs, even
+    existing logs
+  - When a new log is added, scan it in its entirety even if `-all_time`
+    is not specified
+- Add new logs:
+  - Google Icarus
+  - Google Skydiver
+  - StartCom
+  - WoSign
+- Overhaul log processing and auditing logic:
+  - STHs are never deleted unless they can be verified
+  - Multiple unverified STHs can be queued per log, laying groundwork
+    for STH pollination support
+  - New state directory layout; current state directories will be
+    migrated, but migration will be removed in a future version
+  - Persist condensed Merkle Tree state between runs, instead of
+    reconstructing it from consistency proof every time
+- Use a lock file to prevent multiple instances of Cert Spotter from
+  running concurrently (which could clobber the state directory).
+- Minor bug fixes and improvements
+
+## v0.2 (2016-08-25)
+- Suppress duplicate identifiers in output.
+- Fix "EOF" error when running under Go 1.7.
+- Fix bug where hook script could fail silently.
+- Fix compilation under Go 1.5.
+
+## v0.1 (2016-07-27)
+- Initial release.

third_party/certspotter/CONTRIBUTING

@@ -0,0 +1,44 @@
+All contributors to Cert Spotter must accept the Developer Certificate
+of Origin (DCO):
+
+	Developer Certificate of Origin
+	Version 1.1
+
+	Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+	660 York Street, Suite 102,
+	San Francisco, CA 94110 USA
+
+	Everyone is permitted to copy and distribute verbatim copies of this
+	license document, but changing it is not allowed.
+
+
+	Developer's Certificate of Origin 1.1
+
+	By making a contribution to this project, I certify that:
+
+	(a) The contribution was created in whole or in part by me and I
+	    have the right to submit it under the open source license
+	    indicated in the file; or
+
+	(b) The contribution is based upon previous work that, to the best
+	    of my knowledge, is covered under an appropriate open source
+	    license and I have the right under that license to submit that
+	    work with modifications, whether created in whole or in part
+	    by me, under the same open source license (unless I am
+	    permitted to submit under a different license), as indicated
+	    in the file; or
+
+	(c) The contribution was provided directly to me by some other
+	    person who certified (a), (b) or (c) and I have not modified
+	    it.
+
+	(d) I understand and agree that this project and the contribution
+	    are public and that a record of the contribution (including all
+	    personal information I submit with it, including my sign-off) is
+	    maintained indefinitely and may be redistributed consistent with
+	    this project or the open source license(s) involved.
+
+To accept the DCO, add the following line to each commit message with
+your real name and email address (`git commit -s` will do this for you):
+
+	Signed-off-by: Alex Smith <[email protected]>

third_party/certspotter/CTCLOUD_PATCH.md

@@ -0,0 +1,53 @@
+# ctcloud patch to upstream CertSpotter
+
+This directory is a copy of upstream CertSpotter
+(https://github.com/SSLMate/certspotter), pulled in as a local module and
+wired up with a `replace` directive in the root `go.mod`.
+
+It is intentionally a copy (not a git submodule) so the patched tree lives in
+this repository and is compiled by the Docker build with no network access to a
+separate fork. To rebase on a newer upstream release:
+
+1. Replace this directory with the new upstream tree.
+2. Re-apply the two hunks below.
+
+## Changes (only two hunks)
+
+### `monitor/config.go`
+
+Adds an exported `IdentifierSink` interface and a `Config.IdentifierSink`
+field:
+
+```go
+type IdentifierSink interface {
+	HandleIdentifiers(ctx context.Context, identifiers *certspotter.Identifiers)
+}
+```
+
+### `monitor/process.go`
+
+In `processCertificate`, immediately after `certInfo.ParseIdentifiers()`, hand
+the parsed identifiers to the sink and return, bypassing the watchlist,
+certificate-chain retrieval and notification paths:
+
+```go
+if config.IdentifierSink != nil {
+	config.IdentifierSink.HandleIdentifiers(ctx, identifiers)
+	return nil
+}
+```
+
+## Why
+
+The sink gives ctcloud parsed DNS identifiers via a direct in-process Go call:
+
+```text
+CT log entry -> CertSpotter parse -> IdentifierSink.HandleIdentifiers -> classifier
+```
+
+No stdout, no JSON, no files, no hooks, no subprocess. Setting the sink also
+skips issuer-chain fetching and the watchlist machinery, which is exactly the
+I/O and serialization overhead the design forbids.
+
+Upstream is otherwise unmodified. License and notices are preserved in
+`LICENSE`.

third_party/certspotter/LICENSE

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

third_party/certspotter/README.md

@@ -0,0 +1,183 @@
+# Cert Spotter - Certificate Transparency Monitor - Open Source Edition
+
+**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that
+alerts you when an SSL/TLS certificate is issued for one of your domains.
+Cert Spotter is easier to use than other open source CT monitors, since it does not require
+a database. It's also more robust, since it uses a special certificate parser
+that ensures it won't miss certificates.
+
+Cert Spotter is available as a hosted service by SSLMate that also
+provides monitoring for expiring and incorrectly-installed certificates,
+Slack notifications, and a Web dashboard for tracking all of your
+certificates.  Visit <https://sslmate.com/certspotter> to sign up.
+
+You can use Cert Spotter to detect:
+
+ * Certificates issued to attackers who have compromised your DNS and
+   are redirecting your visitors to their malicious site.
+ * Certificates issued to attackers who have taken over an abandoned
+   sub-domain in order to serve malware under your name.
+ * Certificates issued to attackers who have compromised a certificate
+   authority and want to impersonate your site.
+ * Certificates issued in violation of your corporate policy
+   or outside of your centralized certificate procurement process.
+
+## Quickstart
+
+The following instructions require you to have [Go version 1.21 or higher](https://go.dev/dl/) installed.
+
+1. Install the certspotter command using the `go` command:
+
+   ```
+   go install software.sslmate.com/src/certspotter/cmd/certspotter@latest
+   ```
+
+2. Create a watch list file `$HOME/.certspotter/watchlist` containing the DNS names you want to monitor,
+   one per line.  To monitor an entire domain tree (including the domain itself
+   and all sub-domains) prefix the domain name with a dot (e.g. `.example.com`).
+   To monitor a single DNS name only, do not prefix the name with a dot.
+
+3. Place one or more email addresses in the `$HOME/.certspotter/email_recipients`
+   file (one per line), and/or place one or more executable scripts in the
+   `$HOME/.certspotter/hooks.d` directory.  certspotter will email the listed
+   addresses (requires your system to have a working sendmail command) and
+   execute the provided scripts when it detects a certificate for a domain on
+   your watch list.
+
+4. Configure your system to run `certspotter` as a daemon.  You may want to specify
+   the `-start_at_end` command line option to tell certspotter to start monitoring
+   new logs at the end instead of the beginning.  This saves significant bandwidth, but
+   you won't be notified about certificates which were logged before you started
+   using certspotter.
+
+## Documentation
+
+* Command line options and operational details: [certspotter(8) man page](man/certspotter.md)
+* The script interface: [certspotter-script(8) man page](man/certspotter-script.md)
+* Authorizing known certificates: [certspotter-authorize(8) man page](man/certspotter-authorize.md)
+* Monitoring certspotter with Prometheus: [certspotter-metrics(8) man page](man/certspotter-metrics.md)
+* [Change Log](CHANGELOG.md)
+
+## Authorizing Known Certificates to Prevent False Alarms
+
+You can use the **certspotter-authorize** command to tell certspotter
+about legitimate certificates issued by your certificate authority.
+certspotter won't notify you when it discovers an authorized certificate
+(or its corresponding precertificate) in Certificate Transparency logs.
+
+To install certspotter-authorize, run:
+
+```
+go install software.sslmate.com/src/certspotter/cmd/certspotter-authorize@latest
+```
+
+To authorize a certificate, run:
+
+```
+certspotter-authorize /path/to/cert.pem
+```
+
+For more details, see the [certspotter-authorize(8) man page](man/certspotter-authorize.md).
+
+## What certificates are detected by Cert Spotter?
+
+In the default configuration, any certificate that is logged to a
+Certificate Transparency log listed as Qualified, Usable, or ReadOnly
+in the Google Chrome or Apple log lists will be detected by Cert Spotter.
+By default, Chrome, Safari, Firefox, and Edge only accept certificates that are
+logged to at least one such log (Firefox and Edge use the Chrome log list), so any
+certificate that works in these browsers will be detected by Cert Spotter.
+
+Cert Spotter monitors both traditional RFC6962 logs and modern static-ct-api logs.
+
+## Security
+
+Cert Spotter assumes an adversarial model in which an attacker produces
+a certificate that is accepted by at least some clients but goes
+undetected because of an encoding error that prevents CT monitors from
+understanding it.  To defend against this attack, Cert Spotter uses a
+special certificate parser that keeps the certificate unparsed except
+for the identifiers.  If one of the identifiers matches a domain on your
+watchlist, you will be notified, even if other parts of the certificate
+are unparsable.
+
+Cert Spotter takes special precautions to ensure identifiers are parsed
+correctly, and implements defenses against identifier-based attacks.
+For instance, if a DNS identifier contains a null byte, Cert Spotter
+interprets it as two identifiers: the complete identifier, and the
+identifier formed by truncating at the first null byte.  For example, a
+certificate for `example.org\0.example.com` will alert the owners of both
+`example.org` and `example.com`.  This defends against [null prefix attacks](
+http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf).
+
+SSLMate continuously monitors CT logs to make sure every certificate's
+identifiers can be successfully parsed, and will release updates to
+Cert Spotter as necessary to fix parsing failures.
+
+Cert Spotter understands wildcard DNS names, and will alert
+you if a wildcard certificate might match an identifier on
+your watchlist.  For example, a watchlist entry for `sub.example.com` would
+match certificates for `*.example.com`.
+
+Cert Spotter is not just a log monitor, but also a log auditor which
+checks that the log is obeying its append-only property.  A future
+release of Cert Spotter will support gossiping with other log monitors
+to ensure the log is presenting a single view.
+
+## Testing
+
+Every day at around 12:05 UTC, a certificate is issued and logged to CT
+for the DNS name `DAY-OF-WEEK.test.certspotter.org`. You can add one
+or more of these DNS names (or `.test.certspotter.org`) to your watch
+list to get confirmation that Cert Spotter is working.
+
+| DNS Name                         | Issuer        | Key Type |
+| -------------------------------- | ------------- | -------- |
+| `monday.test.certspotter.org`    | Sectigo       | RSA      |
+| `tuesday.test.certspotter.org`   | Let's Encrypt | ECDSA    |
+| `wednesday.test.certspotter.org` | Google        | RSA      |
+| `thursday.test.certspotter.org`  | Sectigo       | ECDSA    |
+| `friday.test.certspotter.org`    | Let's Encrypt | RSA      |
+| `saturday.test.certspotter.org`  | Google        | ECDSA    |
+| `sunday.test.certspotter.org`    | Let's Encrypt | ECDSA    |
+
+## Monitoring certspotter with Prometheus (Experimental)
+
+You can use the experimental **certspotter-metrics** command to output Prometheus metrics
+about certspotter's progress monitoring each CT log, such as how many entries
+have been downloaded and verified. certspotter-metrics reads the certspotter
+state directory and writes metrics to stdout; run it periodically and expose
+the output to Prometheus, e.g. with the node_exporter textfile collector.
+
+To install certspotter-metrics, run:
+
+```
+go install software.sslmate.com/src/certspotter/cmd/certspotter-metrics@latest
+```
+
+Here's a crontab entry that exposes metrics to the node_exporter textfile
+collector every five minutes:
+
+```
+*/5 * * * * root certspotter-metrics > /var/lib/node_exporter/certspotter.prom.tmp && mv /var/lib/node_exporter/certspotter.prom.tmp /var/lib/node_exporter/certspotter.prom
+```
+
+Here's a Prometheus query to determine download backlog (number entries
+in the log that haven't been downloaded yet):
+
+```
+certspotter_log_max_sth_size-certspotter_log_download_position
+```
+
+certspotter-metrics is experimental. Metrics may change in future
+versions, and a future version of certspotter might export metrics
+directly from a builtin HTTP server rather than using a separate command.  Please
+provide feedback at <https://github.com/SSLMate/certspotter/issues/79>.
+
+For more details, see the [certspotter-metrics(8) man page](man/certspotter-metrics.md).
+
+## Copyright
+
+Copyright © 2016-2026 Opsmate, Inc.
+
+Licensed under the [Mozilla Public License Version 2.0](LICENSE).

third_party/certspotter/asn1.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"bytes"
+	"encoding/asn1"
+	"encoding/binary"
+	"errors"
+	"unicode/utf8"
+)
+
+func stringFromByteSlice(chars []byte) string {
+	runes := make([]rune, len(chars))
+	for i, ch := range chars {
+		runes[i] = rune(ch)
+	}
+	return string(runes)
+}
+
+func stringFromUint16Slice(chars []uint16) string {
+	runes := make([]rune, len(chars))
+	for i, ch := range chars {
+		runes[i] = rune(ch)
+	}
+	return string(runes)
+}
+
+func stringFromUint32Slice(chars []uint32) string {
+	runes := make([]rune, len(chars))
+	for i, ch := range chars {
+		runes[i] = rune(ch)
+	}
+	return string(runes)
+}
+
+func decodeASN1String(value *asn1.RawValue) (string, error) {
+	if !value.IsCompound && value.Class == 0 {
+		if value.Tag == 12 {
+			// UTF8String
+			if !utf8.Valid(value.Bytes) {
+				return "", errors.New("malformed UTF8String")
+			}
+			return string(value.Bytes), nil
+		} else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 {
+			// * PrintableString - subset of ASCII
+			// * IA5String - ASCII
+			// * TeletexString - 8 bit charset; not quite ISO-8859-1, but often treated as such
+			// * VisibleString - subset of ASCII
+
+			// Don't enforce character set rules. Allow any 8 bit character, since
+			// CAs routinely mess this up
+			return stringFromByteSlice(value.Bytes), nil
+		} else if value.Tag == 30 {
+			// BMPString - Unicode, encoded in big-endian format using two octets
+			runes := make([]uint16, len(value.Bytes)/2)
+			if err := binary.Read(bytes.NewReader(value.Bytes), binary.BigEndian, runes); err != nil {
+				return "", errors.New("Malformed BMPString: " + err.Error())
+			}
+			return stringFromUint16Slice(runes), nil
+		} else if value.Tag == 28 {
+			// UniversalString - Unicode, encoded in big-endian format using four octets
+			runes := make([]uint32, len(value.Bytes)/4)
+			if err := binary.Read(bytes.NewReader(value.Bytes), binary.BigEndian, runes); err != nil {
+				return "", errors.New("Malformed UniversalString: " + err.Error())
+			}
+			return stringFromUint32Slice(runes), nil
+		}
+	}
+	return "", errors.New("not a string")
+}

third_party/certspotter/asn1time.go

@@ -0,0 +1,257 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"encoding/asn1"
+	"errors"
+	"strconv"
+	"time"
+	"unicode"
+)
+
+const (
+	tagUTCTime         = 23
+	tagGeneralizedTime = 24
+)
+
+func isDigit(b byte) bool {
+	return unicode.IsDigit(rune(b))
+}
+
+func bytesToInt(bytes []byte) (int, error) {
+	return strconv.Atoi(string(bytes))
+}
+
+func parseUTCTime(bytes []byte) (time.Time, error) {
+	var err error
+	var year, month, day int
+	var hour, min, sec int
+	var tz *time.Location
+
+	// YYMMDDhhmm
+	if len(bytes) < 10 {
+		return time.Time{}, errors.New("UTCTime is too short")
+	}
+	year, err = bytesToInt(bytes[0:2])
+	if err != nil {
+		return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+	}
+
+	month, err = bytesToInt(bytes[2:4])
+	if err != nil {
+		return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+	}
+
+	day, err = bytesToInt(bytes[4:6])
+	if err != nil {
+		return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+	}
+
+	hour, err = bytesToInt(bytes[6:8])
+	if err != nil {
+		return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+	}
+
+	min, err = bytesToInt(bytes[8:10])
+	if err != nil {
+		return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+	}
+
+	bytes = bytes[10:]
+
+	// (optional) ss
+	if len(bytes) >= 2 && isDigit(bytes[0]) {
+		sec, err = bytesToInt(bytes[0:2])
+		if err != nil {
+			return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+		}
+		bytes = bytes[2:]
+	}
+
+	// timezone (required but allow it to be omitted, since this is a common error)
+	if len(bytes) >= 1 {
+		if bytes[0] == 'Z' {
+			tz = time.UTC
+			bytes = bytes[1:]
+		} else if bytes[0] == '+' {
+			// +hhmm
+			if len(bytes) < 5 {
+				return time.Time{}, errors.New("UTCTime positive timezone offset is too short")
+			}
+			tzHour, err := bytesToInt(bytes[1:3])
+			if err != nil {
+				return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+			}
+
+			tzMin, err := bytesToInt(bytes[3:5])
+			if err != nil {
+				return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+			}
+
+			tz = time.FixedZone("", tzHour*3600+tzMin*60)
+			bytes = bytes[5:]
+		} else if bytes[0] == '-' {
+			// -hhmm
+			if len(bytes) < 5 {
+				return time.Time{}, errors.New("UTCTime negative timezone offset is too short")
+			}
+			tzHour, err := bytesToInt(bytes[1:3])
+			if err != nil {
+				return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+			}
+
+			tzMin, err := bytesToInt(bytes[3:5])
+			if err != nil {
+				return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error())
+			}
+
+			tz = time.FixedZone("", -1*(tzHour*3600+tzMin*60))
+			bytes = bytes[5:]
+		}
+	} else {
+		tz = time.UTC
+	}
+
+	if len(bytes) > 0 {
+		return time.Time{}, errors.New("UTCTime has trailing garbage")
+	}
+
+	// https://tools.ietf.org/html/rfc5280#section-4.1.2.5.1
+	if year >= 50 {
+		year = 1900 + year
+	} else {
+		year = 2000 + year
+	}
+
+	return time.Date(year, time.Month(month), day, hour, min, sec, 0, tz), nil
+}
+
+func parseGeneralizedTime(bytes []byte) (time.Time, error) {
+	var err error
+	var year, month, day int
+	var hour, min, sec, ms int
+	var tz *time.Location
+
+	// YYYYMMDDHH
+	if len(bytes) < 10 {
+		return time.Time{}, errors.New("GeneralizedTime is too short")
+	}
+	year, err = bytesToInt(bytes[0:4])
+	if err != nil {
+		return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+	}
+
+	month, err = bytesToInt(bytes[4:6])
+	if err != nil {
+		return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+	}
+
+	day, err = bytesToInt(bytes[6:8])
+	if err != nil {
+		return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+	}
+
+	hour, err = bytesToInt(bytes[8:10])
+	if err != nil {
+		return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+	}
+
+	bytes = bytes[10:]
+
+	// (optional) MM
+	if len(bytes) >= 2 && isDigit(bytes[0]) {
+		min, err = bytesToInt(bytes[0:2])
+		if err != nil {
+			return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+		}
+		bytes = bytes[2:]
+		// (optional) SS
+		if len(bytes) >= 2 && isDigit(bytes[0]) {
+			sec, err = bytesToInt(bytes[0:2])
+			if err != nil {
+				return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+			}
+			bytes = bytes[2:]
+			// (optional) .fff
+			if len(bytes) >= 1 && bytes[0] == '.' {
+				if len(bytes) < 4 {
+					return time.Time{}, errors.New("GeneralizedTime fractional seconds is too short")
+				}
+				ms, err = bytesToInt(bytes[1:4])
+				if err != nil {
+					return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+				}
+				bytes = bytes[4:]
+			}
+		}
+	}
+
+	// timezone (Z or +hhmm or -hhmm or nothing)
+	if len(bytes) >= 1 {
+		if bytes[0] == 'Z' {
+			bytes = bytes[1:]
+			tz = time.UTC
+		} else if bytes[0] == '+' {
+			// +hhmm
+			if len(bytes) < 5 {
+				return time.Time{}, errors.New("GeneralizedTime positive timezone offset is too short")
+			}
+			tzHour, err := bytesToInt(bytes[1:3])
+			if err != nil {
+				return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+			}
+
+			tzMin, err := bytesToInt(bytes[3:5])
+			if err != nil {
+				return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+			}
+
+			tz = time.FixedZone("", tzHour*3600+tzMin*60)
+			bytes = bytes[5:]
+		} else if bytes[0] == '-' {
+			// -hhmm
+			if len(bytes) < 5 {
+				return time.Time{}, errors.New("GeneralizedTime negative timezone offset is too short")
+			}
+			tzHour, err := bytesToInt(bytes[1:3])
+			if err != nil {
+				return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+			}
+
+			tzMin, err := bytesToInt(bytes[3:5])
+			if err != nil {
+				return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error())
+			}
+
+			tz = time.FixedZone("", -1*(tzHour*3600+tzMin*60))
+			bytes = bytes[5:]
+		}
+	} else {
+		tz = time.UTC
+	}
+
+	if len(bytes) > 0 {
+		return time.Time{}, errors.New("GeneralizedTime has trailing garbage")
+	}
+
+	return time.Date(year, time.Month(month), day, hour, min, sec, ms*1000*1000, tz), nil
+}
+
+func decodeASN1Time(value *asn1.RawValue) (time.Time, error) {
+	if !value.IsCompound && value.Class == 0 {
+		if value.Tag == tagUTCTime {
+			return parseUTCTime(value.Bytes)
+		} else if value.Tag == tagGeneralizedTime {
+			return parseGeneralizedTime(value.Bytes)
+		}
+	}
+	return time.Time{}, errors.New("not a time value")
+}

third_party/certspotter/asn1time_test.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"testing"
+	"time"
+)
+
+type timeTest struct {
+	in  string
+	ok  bool
+	out time.Time
+}
+
+var utcTimeTests = []timeTest{
+	{"9502101525Z", true, time.Date(1995, time.February, 10, 15, 25, 0, 0, time.UTC)},
+	{"950210152542Z", true, time.Date(1995, time.February, 10, 15, 25, 42, 0, time.UTC)},
+	{"1502101525Z", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)},
+	{"150210152542Z", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)},
+	{"1502101525+1000", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", 10*3600))},
+	{"1502101525-1000", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", -1*(10*3600)))},
+	{"1502101525+1035", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", 10*3600+35*60))},
+	{"1502101525-1035", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", -1*(10*3600+35*60)))},
+	{"150210152542+1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", 10*3600))},
+	{"150210152542-1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", -1*(10*3600)))},
+	{"150210152542+1035", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", 10*3600+35*60))},
+	{"150210152542-1035", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", -1*(10*3600+35*60)))},
+	{"1502101525", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)},
+	{"150210152542", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)},
+	{"", false, time.Time{}},
+	{"123", false, time.Time{}},
+	{"150210152542-10", false, time.Time{}},
+	{"150210152542F", false, time.Time{}},
+	{"150210152542ZF", false, time.Time{}},
+}
+
+func TestUTCTime(t *testing.T) {
+	for i, test := range utcTimeTests {
+		ret, err := parseUTCTime([]byte(test.in))
+		if err != nil {
+			if test.ok {
+				t.Errorf("#%d: parseUTCTime(%q) failed with error %v", i, test.in, err)
+			}
+			continue
+		}
+		if !test.ok {
+			t.Errorf("#%d: parseUTCTime(%q) succeeded, should have failed", i, test.in)
+			continue
+		}
+		if !test.out.Equal(ret) {
+			t.Errorf("#%d: parseUTCTime(%q) = %v, want %v", i, test.in, ret, test.out)
+		}
+	}
+}
+
+var generalizedTimeTests = []timeTest{
+	{"2015021015", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.UTC)},
+	{"201502101525", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)},
+	{"20150210152542", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)},
+	{"20150210152542.123", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.UTC)},
+	{"20150210152542.12", false, time.Time{}},
+	{"20150210152542.1", false, time.Time{}},
+	{"20150210152542.", false, time.Time{}},
+
+	{"2015021015Z", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.UTC)},
+	{"201502101525Z", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)},
+	{"20150210152542Z", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)},
+	{"20150210152542.123Z", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.UTC)},
+	{"20150210152542.12Z", false, time.Time{}},
+	{"20150210152542.1Z", false, time.Time{}},
+	{"20150210152542.Z", false, time.Time{}},
+
+	{"2015021015+1000", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.FixedZone("", 10*3600))},
+	{"201502101525+1000", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", 10*3600))},
+	{"20150210152542+1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", 10*3600))},
+	{"20150210152542.123+1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.FixedZone("", 10*3600))},
+	{"20150210152542.12+1000", false, time.Time{}},
+	{"20150210152542.1+1000", false, time.Time{}},
+	{"20150210152542.+1000", false, time.Time{}},
+
+	{"2015021015-0835", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.FixedZone("", -1*(8*3600+35*60)))},
+	{"201502101525-0835", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", -1*(8*3600+35*60)))},
+	{"20150210152542-0835", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", -1*(8*3600+35*60)))},
+	{"20150210152542.123-0835", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.FixedZone("", -1*(8*3600+35*60)))},
+	{"20150210152542.12-0835", false, time.Time{}},
+	{"20150210152542.1-0835", false, time.Time{}},
+	{"20150210152542.-0835", false, time.Time{}},
+
+	{"", false, time.Time{}},
+	{"123", false, time.Time{}},
+	{"2015021015+1000Z", false, time.Time{}},
+	{"2015021015x", false, time.Time{}},
+	{"201502101525Zf", false, time.Time{}},
+}
+
+func TestGeneralizedTime(t *testing.T) {
+	for i, test := range generalizedTimeTests {
+		ret, err := parseGeneralizedTime([]byte(test.in))
+		if err != nil {
+			if test.ok {
+				t.Errorf("#%d: parseGeneralizedTime(%q) failed with error %v", i, test.in, err)
+			}
+			continue
+		}
+		if !test.ok {
+			t.Errorf("#%d: parseGeneralizedTime(%q) succeeded, should have failed", i, test.in)
+			continue
+		}
+		if !test.out.Equal(ret) {
+			t.Errorf("#%d: parseGeneralizedTime(%q) = %v, want %v", i, test.in, ret, test.out)
+		}
+	}
+}

third_party/certspotter/canonicalize.go

@@ -0,0 +1,92 @@
+// Copyright (C) 2019 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"encoding/asn1"
+)
+
+func canonicalizeRDNString(fromStr string) string {
+	from := []byte(fromStr)
+	to := []byte{}
+	inWhitespace := true
+	for _, ch := range from {
+		if ch == ' ' || ch == '\f' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\v' {
+			if !inWhitespace {
+				to = append(to, ' ')
+			}
+			inWhitespace = true
+		} else {
+			if ch >= 'A' && ch <= 'Z' {
+				to = append(to, ch+32) // convert to lowercase
+			} else {
+				to = append(to, ch)
+			}
+			inWhitespace = false
+		}
+	}
+	if inWhitespace && len(to) > 0 {
+		// whack off the space character that we appended
+		to = to[:len(to)-1]
+	}
+	return string(to)
+}
+
+func shouldCanonicalizeASN1String(value *asn1.RawValue) bool {
+	if !value.IsCompound && value.Class == 0 {
+		return value.Tag == 12 || value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 || value.Tag == 30 || value.Tag == 28
+	}
+	return false
+}
+
+func canonicalizeATV(oldATV AttributeTypeAndValue) (AttributeTypeAndValue, error) {
+	if shouldCanonicalizeASN1String(&oldATV.Value) {
+		str, err := decodeASN1String(&oldATV.Value)
+		if err != nil {
+			return AttributeTypeAndValue{}, err
+		}
+		str = canonicalizeRDNString(str)
+		return AttributeTypeAndValue{
+			Type: oldATV.Type,
+			Value: asn1.RawValue{
+				Class:      0,
+				Tag:        asn1.TagUTF8String,
+				IsCompound: false,
+				Bytes:      []byte(str),
+			},
+		}, nil
+	} else {
+		return oldATV, nil
+	}
+}
+
+func canonicalizeRDNSet(oldSet RelativeDistinguishedNameSET) (RelativeDistinguishedNameSET, error) {
+	newSet := make([]AttributeTypeAndValue, len(oldSet))
+	for i := range oldSet {
+		var err error
+		newSet[i], err = canonicalizeATV(oldSet[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	return newSet, nil
+}
+
+func CanonicalizeRDNSequence(oldSequence RDNSequence) (RDNSequence, error) {
+	newSequence := make([]RelativeDistinguishedNameSET, len(oldSequence))
+	for i := range oldSequence {
+		var err error
+		newSequence[i], err = canonicalizeRDNSet(oldSequence[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	return newSequence, nil
+}

third_party/certspotter/canonicalize_test.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2019 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"testing"
+)
+
+type stringCanonTest struct {
+	in  string
+	out string
+}
+
+var stringCanonTests = []stringCanonTest{
+	{"", ""},
+	{" ", ""},
+	{"  ", ""},
+	{"abc", "abc"},
+	{"aBc", "abc"},
+	{"ab c", "ab c"},
+	{"ab  c", "ab c"},
+	{"ab\n c", "ab c"},
+	{" ab c ", "ab c"},
+	{" ab  c ", "ab c"},
+	{" ab  c", "ab c"},
+	{"ab  c ", "ab c"},
+	{"abc ", "abc"},
+	{"abc  ", "abc"},
+	{"  abc  ", "abc"},
+	{"  abc ", "abc"},
+	{"  abc", "abc"},
+	{"  aBc de  f      g\n", "abc de f g"},
+}
+
+func TestCanonicalizeRDNString(t *testing.T) {
+	for i, test := range stringCanonTests {
+		ret := canonicalizeRDNString(test.in)
+		if test.out != ret {
+			t.Errorf("#%d: canonicalizeRDNString(%q) = %q, want %q", i, test.in, ret, test.out)
+		}
+	}
+}

third_party/certspotter/cmd/certspotter-authorize/.gitignore

@@ -0,0 +1 @@
+/certspotter-authorize

third_party/certspotter/cmd/certspotter-authorize/main.go

@@ -0,0 +1,202 @@
+// Copyright (C) 2026 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"strconv"
+	"strings"
+
+	"software.sslmate.com/src/certspotter"
+)
+
+var programName = os.Args[0]
+var Version = "unknown"
+var Source = "unknown"
+
+func certspotterVersion() (string, string) {
+	if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") {
+		return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path
+	} else {
+		return Version, Source
+	}
+}
+
+func homedir() string {
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		panic(fmt.Errorf("unable to determine home directory: %w", err))
+	}
+	return homedir
+}
+
+func startedBySupervisor() bool {
+	return os.Getenv("SYSTEMD_EXEC_PID") == strconv.Itoa(os.Getpid())
+}
+
+func defaultStateDir() string {
+	if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
+		return envVar
+	} else if envVar := os.Getenv("STATE_DIRECTORY"); envVar != "" && startedBySupervisor() {
+		return envVar
+	} else {
+		return filepath.Join(homedir(), ".certspotter")
+	}
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}
+
+func readCertFile(path string) ([]byte, error) {
+	if path == "-" {
+		return io.ReadAll(os.Stdin)
+	} else {
+		return os.ReadFile(path)
+	}
+}
+
+func parseCertificate(certBytes []byte) ([]byte, error) {
+	block, _ := pem.Decode(certBytes)
+	if block != nil {
+		if block.Type == "CERTIFICATE" {
+			return block.Bytes, nil
+		}
+		return nil, fmt.Errorf("PEM block type is %q, expected CERTIFICATE", block.Type)
+	}
+	return nil, fmt.Errorf("no PEM data found")
+}
+
+func computeTBSHash(certDER []byte) ([32]byte, error) {
+	certInfo, err := certspotter.MakeCertInfoFromRawCert(certDER)
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("error parsing certificate: %w", err)
+	}
+	precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS)
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err)
+	}
+	return sha256.Sum256(precertTBS.Raw), nil
+}
+
+func createNotifiedMarker(stateDir string, tbsHash [32]byte) (string, error) {
+	tbsHex := hex.EncodeToString(tbsHash[:])
+
+	certsDir := filepath.Join(stateDir, "certs")
+	tbsDir := filepath.Join(certsDir, tbsHex[0:2])
+	notifiedPath := filepath.Join(tbsDir, "."+tbsHex+".notified")
+
+	// Check if already notified
+	if fileExists(notifiedPath) {
+		return notifiedPath, nil
+	}
+
+	// Create certs directory if needed
+	if err := os.Mkdir(certsDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+		return "", fmt.Errorf("error creating certs directory: %w", err)
+	}
+
+	// Create TBS-specific subdirectory if needed
+	if err := os.Mkdir(tbsDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+		return "", fmt.Errorf("error creating directory: %w", err)
+	}
+
+	// Create marker file
+	if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
+		return "", fmt.Errorf("error creating marker file: %w", err)
+	}
+
+	return notifiedPath, nil
+}
+
+func main() {
+	version, source := certspotterVersion()
+
+	var flags struct {
+		stateDir  string
+		printhash bool
+		version   bool
+	}
+
+	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "State directory used by certspotter")
+	flag.BoolVar(&flags.printhash, "printhash", false, "Instead of authorizing certificate, print its TBS hash")
+	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
+	flag.Parse()
+
+	if flags.version {
+		fmt.Fprintf(os.Stdout, "certspotter-authorize version %s (%s)\n", version, source)
+		os.Exit(0)
+	}
+
+	args := flag.Args()
+	if len(args) == 0 {
+		fmt.Fprintf(os.Stderr, "Usage: %s [options] -|CERTFILE...\n", programName)
+		fmt.Fprintf(os.Stderr, "Purpose: suppress future certspotter notifications for a certificate and its corresponding precertificate.\n")
+		fmt.Fprintf(os.Stderr, "Options:\n")
+		flag.PrintDefaults()
+		os.Exit(2)
+	}
+
+	if len(args) > 1 {
+		for _, arg := range args {
+			if arg == "-" {
+				fmt.Fprintf(os.Stderr, "%s: '-' must be the only argument when used\n", programName)
+				os.Exit(2)
+			}
+		}
+	}
+
+	for _, certPath := range args {
+		certBytes, err := readCertFile(certPath)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: error reading certificate: %s\n", programName, err)
+			os.Exit(1)
+		}
+
+		certDER, err := parseCertificate(certBytes)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %s: %s\n", programName, certPath, err)
+			os.Exit(1)
+		}
+
+		tbsHash, err := computeTBSHash(certDER)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %s: %s\n", programName, certPath, err)
+			os.Exit(1)
+		}
+
+		if flags.printhash {
+			if certPath == "-" {
+				fmt.Println(hex.EncodeToString(tbsHash[:]))
+			} else {
+				fmt.Printf("%s  %s\n", hex.EncodeToString(tbsHash[:]), certPath)
+			}
+		} else {
+			_, err = createNotifiedMarker(flags.stateDir, tbsHash)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
+				os.Exit(1)
+			}
+		}
+	}
+
+	os.Exit(0)
+}

third_party/certspotter/cmd/certspotter-authorize/main_test.go

@@ -0,0 +1,190 @@
+// Copyright (C) 2026 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package main
+
+import (
+	"bytes"
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const testCertPEM = `
+-----BEGIN CERTIFICATE-----
+MIIGuzCCBSOgAwIBAgIRANubk4g/6c+TF8jITzhFX44wDQYJKoZIhvcNAQELBQAw
+YDELMAkGA1UEBhMCR0IxGDAWBgNVBAoTD1NlY3RpZ28gTGltaXRlZDE3MDUGA1UE
+AxMuU2VjdGlnbyBQdWJsaWMgU2VydmVyIEF1dGhlbnRpY2F0aW9uIENBIERWIFIz
+NjAeFw0yNTEyMDIwMDAwMDBaFw0yNjExMjEyMzU5NTlaMBYxFDASBgNVBAMTC3Nz
+bG1hdGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA235/3Y/E
+4yPAHPa37C7Fgp7KPVjjuTB5vKV9nYIJzfp7NgvDBlf7k5bZFCSsSIj2txhL0hzX
+Bwvmy7u7CYR7CApr2Rx2UPOl7Gmlt/DmtfyKac8Iunn2ozuGZDtxq19Go4NL9jl9
+e9O3H/lcL/ZFqzbUNlKIOfkOYkOxM3qpQXHTXuhkeI2MJO/S4wX8y8/8uhArWQ9e
+h/YrtJlO9fla60kLUlQF7mtJTc+0oB3+N4eF5t2a8Pav00T6lVvH8hMhbY0nZ/tB
+CD6/I6yelh8cP094VRJEGWs+zcEuXpz4FsZggkhF/l+AhQ+DfgxZhno4M60kBKC8
+Un1BTGX5TjfjJQIDAQABo4IDODCCAzQwHwYDVR0jBBgwFoAUaMASFhgOr872h6Yy
+V6NGUV3LBycwHQYDVR0OBBYEFKg2kl8xIzdSxjOEAzlNpdpigAazMA4GA1UdDwEB
+/wQEAwIFoDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMEkGA1Ud
+IARCMEAwNAYLKwYBBAGyMQECAgcwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9zZWN0
+aWdvLmNvbS9DUFMwCAYGZ4EMAQIBMIGEBggrBgEFBQcBAQR4MHYwTwYIKwYBBQUH
+MAKGQ2h0dHA6Ly9jcnQuc2VjdGlnby5jb20vU2VjdGlnb1B1YmxpY1NlcnZlckF1
+dGhlbnRpY2F0aW9uQ0FEVlIzNi5jcnQwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3Nw
+LnNlY3RpZ28uY29tMIIBfQYKKwYBBAHWeQIEAgSCAW0EggFpAWcAdQDXbX0Q0af1
+d8LH6V/XAL/5gskzWmXh0LMBcxfAyMVpdwAAAZrg2UujAAAEAwBGMEQCIApVmNqo
+gzJBNbcVXezO5sSvOFE5FaVZVz/eaqnCbG+2AiAT/A7XPtOYHwsE0wmTUBCTV/0l
+bF7lk573b3rNtBvP3QB2AK9niDtXsE7dj6bZfvYuqOuBCsdxYPAkXlXWDC/nhYc6
+AAABmuDZTFoAAAQDAEcwRQIgODruJKtbjW1QJcQP7ARZAw5FgChfI599pJBA2bbQ
+suICIQCskIUnzQCD6taycnQCN3zpu+rsz3Vd4AsMeFDM/cDJ5AB2AKyrMHBs6+yE
+MfQT0vSRXxEeQiRDsfKmjE88KzunHgLDAAABmuDZS5kAAAQDAEcwRQIgD9IQCPTc
+N88jbz5DUILwmDruTo411Ep5M2ZryNjBkywCIQCFwIyqGZEd+PiFv4l+5LOV3yDW
+/zUuimFUoAJH5OIiNDBsBgNVHREEZTBjggtzc2xtYXRlLmNvbYIbKi5odHRwLWFw
+cHJvdmFsLnNzbG1hdGUuY29tghFjZXJ0cy5zc2xtYXRlLmNvbYITY29uc29sZS5z
+c2xtYXRlLmNvbYIPd3d3LnNzbG1hdGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBgQBR
+Pjx14qo9PiYYEE1695CHdctA6up8L+n0MRapZcxALN/cetfGeoR00ZEH+7b1X7Ma
+F9GGv1OtJXDoCySlAsdwFKHYtKhrUYRuQXdKGkTjdMzKO/+5kXeZIqgsCR10j8nr
+Zq0Zcg2ply4j03/0y7+8ZNC1Erp4DB1Tq7ybgXnyURaNQTHsSkDoxMT/bWIrhGD0
+C8kN/ExkFvOBQlzdbuwo2d3v0zSM4mYmnqUhUYHprZllOziYgxIqjM/7mfnDkVAi
+ov8yNJtn6EPt1wt6Oo3fC+Ft1T/kbSxeZbqWf3Zgbon5ijmNz+xqkb8br2+JdzM+
+8gEIqO6mNoMl0tayzb4a5KDaHxhczMGB3ggBwpVcdLtYBBa41thrgRP0VARqFTFG
+IIkC9gPMjScf+uv9CQPsNk3kFI8vN4T3x4/g54N8Mc3M4JxvLaOsBj8dMeyq7v2p
+1zE9WRngMUWuPgx0O94c0Pteumg/+pSGVeRapIuYZxXvkmLJ5wmwgYepix+cw1w=
+-----END CERTIFICATE-----
+`
+
+func TestComputeTBSHash(t *testing.T) {
+	certDER, err := parseCertificate([]byte(testCertPEM))
+	if err != nil {
+		t.Fatalf("parseCertificate failed: %v", err)
+	}
+	tbsHash, err := computeTBSHash(certDER)
+	if err != nil {
+		t.Fatalf("computeTBSHash failed: %v", err)
+	}
+	if expected := [...]byte{0x3c, 0xf6, 0xb2, 0x44, 0xc2, 0x95, 0x85, 0xdb, 0xfb, 0xfd, 0x42, 0x0a, 0x6a, 0x4c, 0x62, 0xf7, 0x96, 0x8f, 0xa9, 0x05, 0xb4, 0xd6, 0xa4, 0xf5, 0x9d, 0x4d, 0x3b, 0xc9, 0xfa, 0xcb, 0x0c, 0xc8}; expected != tbsHash {
+		t.Fatalf("computeTBSHash returned %x; expected %x", tbsHash, expected)
+	}
+}
+
+func TestCreateNotifiedMarker(t *testing.T) {
+	stateDir := t.TempDir()
+
+	certDER, err := parseCertificate([]byte(testCertPEM))
+	if err != nil {
+		t.Fatalf("parseCertificate failed: %v", err)
+	}
+
+	tbsHash, err := computeTBSHash(certDER)
+	if err != nil {
+		t.Fatalf("computeTBSHash failed: %v", err)
+	}
+
+	// First call should create the marker
+	notifiedPath, err := createNotifiedMarker(stateDir, tbsHash)
+	if err != nil {
+		t.Fatalf("createNotifiedMarker failed: %v", err)
+	}
+
+	// Verify marker file exists
+	if !fileExists(notifiedPath) {
+		t.Fatalf("marker file does not exist: %s", notifiedPath)
+	}
+
+	// Verify path structure is correct
+	tbsHex := hex.EncodeToString(tbsHash[:])
+	expectedPath := filepath.Join(stateDir, "certs", tbsHex[0:2], "."+tbsHex+".notified")
+	if notifiedPath != expectedPath {
+		t.Fatalf("unexpected marker path: got %s, expected %s", notifiedPath, expectedPath)
+	}
+
+	// Second call should succeed (idempotency)
+	notifiedPath2, err := createNotifiedMarker(stateDir, tbsHash)
+	if err != nil {
+		t.Fatalf("createNotifiedMarker second call failed: %v", err)
+	}
+	if notifiedPath != notifiedPath2 {
+		t.Fatalf("second call returned different path: got %s, expected %s", notifiedPath2, notifiedPath)
+	}
+}
+
+func TestReadCertFile(t *testing.T) {
+	// Test reading from a file
+	tmpDir := t.TempDir()
+	certPath := filepath.Join(tmpDir, "cert.pem")
+	if err := os.WriteFile(certPath, []byte(testCertPEM), 0644); err != nil {
+		t.Fatalf("failed to write test cert: %v", err)
+	}
+
+	certBytes, err := readCertFile(certPath)
+	if err != nil {
+		t.Fatalf("readCertFile failed: %v", err)
+	}
+	if !bytes.Equal(certBytes, []byte(testCertPEM)) {
+		t.Fatal("readCertFile returned different content")
+	}
+}
+
+func TestFileExists(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	// Test with non-existent file
+	if fileExists(filepath.Join(tmpDir, "nonexistent")) {
+		t.Fatal("fileExists returned true for non-existent file")
+	}
+
+	// Test with existing file
+	existingFile := filepath.Join(tmpDir, "existing")
+	if err := os.WriteFile(existingFile, []byte("test"), 0644); err != nil {
+		t.Fatalf("failed to write test file: %v", err)
+	}
+	if !fileExists(existingFile) {
+		t.Fatal("fileExists returned false for existing file")
+	}
+}
+
+func TestEndToEnd(t *testing.T) {
+	stateDir := t.TempDir()
+
+	certDER, err := parseCertificate([]byte(testCertPEM))
+	if err != nil {
+		t.Fatalf("parseCertificate failed: %v", err)
+	}
+
+	tbsHash, err := computeTBSHash(certDER)
+	if err != nil {
+		t.Fatalf("computeTBSHash failed: %v", err)
+	}
+
+	notifiedPath, err := createNotifiedMarker(stateDir, tbsHash)
+	if err != nil {
+		t.Fatalf("createNotifiedMarker failed: %v", err)
+	}
+
+	// Verify the marker file structure matches what monitor/fsstate.go expects
+	tbsHex := hex.EncodeToString(tbsHash[:])
+	expectedDir := filepath.Join(stateDir, "certs", tbsHex[0:2])
+	expectedFile := filepath.Join(expectedDir, "."+tbsHex+".notified")
+
+	if notifiedPath != expectedFile {
+		t.Fatalf("unexpected marker path: got %s, expected %s", notifiedPath, expectedFile)
+	}
+
+	if !fileExists(expectedFile) {
+		t.Fatalf("marker file does not exist: %s", expectedFile)
+	}
+
+	// Verify file is empty (as expected by certspotter)
+	stat, err := os.Stat(expectedFile)
+	if err != nil {
+		t.Fatalf("failed to stat marker file: %v", err)
+	}
+	if stat.Size() != 0 {
+		t.Fatalf("marker file should be empty, but has size %d", stat.Size())
+	}
+}

third_party/certspotter/cmd/certspotter-metrics/.gitignore

@@ -0,0 +1 @@
+/certspotter-metrics

third_party/certspotter/cmd/certspotter-metrics/main.go

@@ -0,0 +1,245 @@
+// Copyright (C) 2026 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"slices"
+	"strconv"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/monitor"
+)
+
+var programName = os.Args[0]
+var Version = "unknown"
+var Source = "unknown"
+
+func certspotterVersion() (string, string) {
+	if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") {
+		return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path
+	} else {
+		return Version, Source
+	}
+}
+
+func homedir() string {
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		panic(fmt.Errorf("unable to determine home directory: %w", err))
+	}
+	return homedir
+}
+func startedBySupervisor() bool {
+	return os.Getenv("SYSTEMD_EXEC_PID") == strconv.Itoa(os.Getpid())
+}
+func defaultStateDir() string {
+	if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
+		return envVar
+	} else if envVar := os.Getenv("STATE_DIRECTORY"); envVar != "" && startedBySupervisor() {
+		return envVar
+	} else {
+		return filepath.Join(homedir(), ".certspotter")
+	}
+}
+
+// logMetrics is the set of metric values for a single log.
+type logMetrics struct {
+	logID monitor.LogID
+
+	maxSTHExists         bool
+	maxSTHSize           uint64
+	maxSTHTimestamp      uint64
+	downloadPosition     uint64
+	verifiedPosition     uint64
+	verifiedSTHExists    bool
+	verifiedSTHTimestamp uint64
+	unverifiedSTHs       uint64
+	malformedEntries     uint64
+	healthCheckFailures  uint64
+}
+
+// metricFamily describes one of the metrics output by certspotter-metrics.
+// All metrics are gauges labeled with the ID of the log they pertain to.
+type metricFamily struct {
+	name  string
+	help  string
+	value func(*logMetrics) (uint64, bool)
+}
+
+var metricFamilies = []metricFamily{
+	{
+		name:  "certspotter_log_max_sth_size",
+		help:  "Tree size of largest known STH.",
+		value: func(m *logMetrics) (uint64, bool) { return m.maxSTHSize, m.maxSTHExists },
+	},
+	{
+		name:  "certspotter_log_max_sth_timestamp",
+		help:  "Timestamp (in milliseconds since the Epoch) of largest known STH.",
+		value: func(m *logMetrics) (uint64, bool) { return m.maxSTHTimestamp, m.maxSTHExists },
+	},
+	{
+		name:  "certspotter_log_download_position",
+		help:  "Number of entries downloaded from the log, including entries not yet verified.",
+		value: func(m *logMetrics) (uint64, bool) { return m.downloadPosition, true },
+	},
+	{
+		name:  "certspotter_log_verified_position",
+		help:  "Number of entries downloaded from the log and verified against an STH.",
+		value: func(m *logMetrics) (uint64, bool) { return m.verifiedPosition, true },
+	},
+	{
+		name:  "certspotter_log_verified_sth_timestamp",
+		help:  "Timestamp (in milliseconds since the Epoch) of the largest verified STH (the STH whose size is verified_position).",
+		value: func(m *logMetrics) (uint64, bool) { return m.verifiedSTHTimestamp, m.verifiedSTHExists },
+	},
+	{
+		name:  "certspotter_log_unverified_sths",
+		help:  "Number of STHs pending verification.",
+		value: func(m *logMetrics) (uint64, bool) { return m.unverifiedSTHs, true },
+	},
+	{
+		name:  "certspotter_log_malformed_entries",
+		help:  "Number of malformed entries recorded for the log.",
+		value: func(m *logMetrics) (uint64, bool) { return m.malformedEntries, true },
+	},
+	{
+		name:  "certspotter_log_health_check_failures",
+		help:  "Number of health check failures recorded for the log.",
+		value: func(m *logMetrics) (uint64, bool) { return m.healthCheckFailures, true },
+	},
+}
+
+// gatherMetrics computes the metrics for every log that has a state directory.
+// The returned slice is sorted by log ID.
+func gatherMetrics(ctx context.Context, state *monitor.FilesystemState) ([]logMetrics, error) {
+	logIDs, err := state.ListLogs(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("error listing logs in state directory: %w", err)
+	}
+	slices.SortFunc(logIDs, monitor.LogID.Compare)
+
+	allMetrics := make([]logMetrics, 0, len(logIDs))
+	for _, logID := range logIDs {
+		m := logMetrics{logID: logID}
+
+		logState, err := state.LoadLogState(ctx, logID)
+		if err != nil {
+			return nil, fmt.Errorf("error loading state of log %s: %w", logID.Base64String(), err)
+		}
+		largestSTH, err := state.LoadLargestSTH(ctx, logID)
+		if err != nil {
+			return nil, fmt.Errorf("error loading STHs of log %s: %w", logID.Base64String(), err)
+		}
+
+		if largestSTH != nil {
+			m.maxSTHExists = true
+			m.maxSTHSize = largestSTH.TreeSize
+			m.maxSTHTimestamp = largestSTH.Timestamp
+		} else if logState != nil && logState.VerifiedSTH != nil {
+			m.maxSTHExists = true
+			m.maxSTHSize = logState.VerifiedSTH.TreeSize
+			m.maxSTHTimestamp = logState.VerifiedSTH.Timestamp
+		}
+
+		if logState != nil {
+			if logState.DownloadPosition != nil {
+				m.downloadPosition = logState.DownloadPosition.Size()
+			}
+			if logState.VerifiedPosition != nil {
+				m.verifiedPosition = logState.VerifiedPosition.Size()
+			}
+			if logState.VerifiedSTH != nil {
+				m.verifiedSTHExists = true
+				m.verifiedSTHTimestamp = logState.VerifiedSTH.Timestamp
+			}
+		}
+
+		if unverifiedSTHs, err := state.CountSTHs(ctx, logID); err != nil {
+			return nil, fmt.Errorf("error counting STHs of log %s: %w", logID.Base64String(), err)
+		} else {
+			m.unverifiedSTHs = uint64(unverifiedSTHs)
+		}
+		if malformedEntries, err := state.CountMalformedEntries(ctx, logID); err != nil {
+			return nil, fmt.Errorf("error counting malformed entries of log %s: %w", logID.Base64String(), err)
+		} else {
+			m.malformedEntries = uint64(malformedEntries)
+		}
+		if healthCheckFailures, err := state.CountHealthCheckFailures(ctx, logID); err != nil {
+			return nil, fmt.Errorf("error counting health check failures of log %s: %w", logID.Base64String(), err)
+		} else {
+			m.healthCheckFailures = uint64(healthCheckFailures)
+		}
+
+		allMetrics = append(allMetrics, m)
+	}
+	return allMetrics, nil
+}
+
+// writeMetrics writes allMetrics to w in the Prometheus text exposition format.
+func writeMetrics(w io.Writer, allMetrics []logMetrics) error {
+	buf := bufio.NewWriter(w)
+	for _, family := range metricFamilies {
+		if _, err := fmt.Fprintf(buf, "# HELP %s %s\n", family.name, family.help); err != nil {
+			return err
+		}
+		if _, err := fmt.Fprintf(buf, "# TYPE %s gauge\n", family.name); err != nil {
+			return err
+		}
+		for i := range allMetrics {
+			value, include := family.value(&allMetrics[i])
+			if !include {
+				continue
+			}
+			if _, err := fmt.Fprintf(buf, "%s{log_id=\"%s\"} %d\n", family.name, allMetrics[i].logID.Base64String(), value); err != nil {
+				return err
+			}
+		}
+	}
+	return buf.Flush()
+}
+
+func main() {
+	version, source := certspotterVersion()
+
+	var flags struct {
+		stateDir string
+		version  bool
+	}
+	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory where certspotter stores state. This should be the same directory used by certspotter(8)")
+	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
+	flag.Parse()
+
+	if flags.version {
+		fmt.Fprintf(os.Stdout, "certspotter-metrics version %s (%s)\n", version, source)
+		os.Exit(0)
+	}
+
+	state := &monitor.FilesystemState{
+		StateDir: flags.stateDir,
+	}
+
+	allMetrics, err := gatherMetrics(context.Background(), state)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
+		os.Exit(1)
+	}
+	if err := writeMetrics(os.Stdout, allMetrics); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: error writing metrics: %s\n", programName, err)
+		os.Exit(1)
+	}
+}

third_party/certspotter/cmd/certspotter/.gitignore

@@ -0,0 +1 @@
+/certspotter

third_party/certspotter/cmd/certspotter/main.go

@@ -0,0 +1,290 @@
+// Copyright (C) 2016, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"software.sslmate.com/src/certspotter/ctclient"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/monitor"
+)
+
+var programName = os.Args[0]
+var Version = "unknown"
+var Source = "unknown"
+
+const defaultLogList = "https://loglist.certspotter.org/monitor.json"
+
+func certspotterVersion() (string, string) {
+	if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") {
+		return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path
+	} else {
+		return Version, Source
+	}
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}
+func homedir() string {
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		panic(fmt.Errorf("unable to determine home directory: %w", err))
+	}
+	return homedir
+}
+func userCacheDir() string {
+	userCacheDir, err := os.UserCacheDir()
+	if err != nil {
+		panic(fmt.Errorf("unable to determine user cache directory: %w", err))
+	}
+	return userCacheDir
+}
+func startedBySupervisor() bool {
+	return os.Getenv("SYSTEMD_EXEC_PID") == strconv.Itoa(os.Getpid())
+}
+func defaultStateDir() string {
+	if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
+		return envVar
+	} else if envVar := os.Getenv("STATE_DIRECTORY"); envVar != "" && startedBySupervisor() {
+		return envVar
+	} else {
+		return filepath.Join(homedir(), ".certspotter")
+	}
+}
+func defaultConfigDir() string {
+	if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" {
+		return envVar
+	} else if envVar := os.Getenv("CONFIGURATION_DIRECTORY"); envVar != "" && startedBySupervisor() {
+		return envVar
+	} else {
+		return filepath.Join(homedir(), ".certspotter")
+	}
+}
+func defaultCacheDir() string {
+	if envVar := os.Getenv("CERTSPOTTER_CACHE_DIR"); envVar != "" {
+		return envVar
+	} else if envVar := os.Getenv("CACHE_DIRECTORY"); envVar != "" && startedBySupervisor() {
+		return envVar
+	} else {
+		return filepath.Join(userCacheDir(), "certspotter")
+	}
+}
+func defaultWatchListPath() string {
+	return filepath.Join(defaultConfigDir(), "watchlist")
+}
+func defaultWatchListPathIfExists() string {
+	if fileExists(defaultWatchListPath()) {
+		return defaultWatchListPath()
+	} else {
+		return ""
+	}
+}
+func defaultScriptDir() string {
+	return filepath.Join(defaultConfigDir(), "hooks.d")
+}
+func defaultEmailFile() string {
+	return filepath.Join(defaultConfigDir(), "email_recipients")
+}
+
+func simplifyError(err error) error {
+	var pathErr *fs.PathError
+	if errors.As(err, &pathErr) {
+		return pathErr.Err
+	}
+
+	return err
+}
+
+func readWatchListFile(filename string) (monitor.WatchList, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, simplifyError(err)
+	}
+	defer file.Close()
+	return monitor.ReadWatchList(file)
+}
+
+func readEmailFile(filename string) ([]string, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, simplifyError(err)
+	}
+	defer file.Close()
+
+	var emails []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+		emails = append(emails, line)
+	}
+	return emails, scanner.Err()
+}
+
+func appendFunc(slice *[]string) func(string) error {
+	return func(value string) error {
+		*slice = append(*slice, value)
+		return nil
+	}
+}
+
+func main() {
+	version, source := certspotterVersion()
+
+	ctclient.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s; %s; +https://github.com/SSLMate/certspotter)", version, source, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+	loglist.UserAgent = ctclient.UserAgent
+
+	var flags struct {
+		email       []string
+		healthcheck time.Duration
+		logs        string
+		noSave      bool
+		script      string
+		startAtEnd  bool
+		stateDir    string
+		stdout      bool
+		verbose     bool
+		version     bool
+		watchlist   string
+	}
+	flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
+	flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check")
+	flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
+	flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
+	flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
+	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
+	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
+	flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
+	flag.BoolVar(&flags.verbose, "verbose", false, "Print detailed information about certspotter's operation to stderr")
+	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
+	flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch")
+	flag.Parse()
+
+	if flags.version {
+		fmt.Fprintf(os.Stdout, "certspotter version %s (%s)\n", version, source)
+		if flags.verbose {
+			if buildinfo, ok := debug.ReadBuildInfo(); ok {
+				fmt.Fprint(os.Stdout, buildinfo)
+			} else {
+				fmt.Fprint(os.Stdout, "build info not available\n")
+			}
+		}
+		os.Exit(0)
+	}
+	if flags.watchlist == "" {
+		fmt.Fprintf(os.Stderr, "%s: watch list not found: please create %s or specify alternative path using -watchlist\n", programName, defaultWatchListPath())
+		os.Exit(2)
+	}
+
+	fsstate := &monitor.FilesystemState{
+		StateDir:  flags.stateDir,
+		CacheDir:  defaultCacheDir(),
+		SaveCerts: !flags.noSave,
+		Script:    flags.script,
+		ScriptDir: defaultScriptDir(),
+		Email:     flags.email,
+		Stdout:    flags.stdout,
+	}
+	config := &monitor.Config{
+		LogListSource:       flags.logs,
+		State:               fsstate,
+		StartAtEnd:          flags.startAtEnd,
+		Verbose:             flags.verbose,
+		HealthCheckInterval: flags.healthcheck,
+	}
+
+	emailFileExists := false
+	if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
+		emailFileExists = true
+		fsstate.Email = append(fsstate.Email, emailRecipients...)
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
+		os.Exit(1)
+	}
+
+	if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
+		fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
+		fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
+		fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
+		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
+		fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
+		fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
+		fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")
+		os.Exit(2)
+	}
+
+	if flags.watchlist == "-" {
+		watchlist, err := monitor.ReadWatchList(os.Stdin)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err)
+			os.Exit(1)
+		}
+		config.WatchList = watchlist
+	} else {
+		watchlist, err := readWatchListFile(flags.watchlist)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err)
+			os.Exit(1)
+		}
+		config.WatchList = watchlist
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	go func() {
+		ticker := time.NewTicker(24*time.Hour)
+		defer ticker.Stop()
+		for {
+			fsstate.PruneOldErrors()
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+
+	if startedBySupervisor() {
+		// Prevent any process that we spawn from erroneously writing to our directories
+		os.Unsetenv("CACHE_DIRECTORY")
+		os.Unsetenv("CONFIGURATION_DIRECTORY")
+		os.Unsetenv("STATE_DIRECTORY")
+	}
+
+	if err := monitor.Run(ctx, config); ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
+		if flags.verbose {
+			fmt.Fprintf(os.Stderr, "%s: exiting due to SIGINT or SIGTERM\n", programName)
+		}
+		os.Exit(0)
+	} else {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
+		os.Exit(1)
+	}
+}

third_party/certspotter/ctclient/client.go

@@ -0,0 +1,204 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+// Package ctclient implements a client for monitoring RFC6962 and static-ct-api Certificate Transparency logs
+package ctclient
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+var UserAgent = ""
+
+// Create an HTTP client suitable for communicating with CT logs.  dialContext, if non-nil, is used for dialing.
+func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn, error)) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy:                 http.ProxyFromEnvironment,
+			TLSHandshakeTimeout:   15 * time.Second,
+			ResponseHeaderTimeout: 30 * time.Second,
+			MaxIdleConnsPerHost:   10,
+			IdleConnTimeout:       90 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSClientConfig: &tls.Config{
+				// We have to disable TLS certificate validation because because several logs
+				// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
+				// Since we verify that every response we receive from the log is signed
+				// by the log's CT public key (either directly, or indirectly via the Merkle Tree),
+				// TLS certificate validation is not actually necessary.  (We don't want to manage
+				// our own trust store because that adds undesired complexity and would require
+				// updating should a log ever change to a different CA.)
+				InsecureSkipVerify: true,
+			},
+			DialContext:       dialContext,
+			ForceAttemptHTTP2: true,
+		},
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return errors.New("redirects not followed")
+		},
+		Timeout: 60 * time.Second,
+	}
+}
+
+var defaultHTTPClient = NewHTTPClient(nil)
+
+// maxResponseBytes is a generous upper bound on the size of an HTTP response
+// body that we are willing to buffer in memory.
+const maxResponseBytes = 64 << 20 // 64 MiB
+
+// maxErrorResponseBytes is longest error response that we'll read
+const maxErrorResponseBytes = 8 << 10 // 8 KiB
+
+// readResponseBody reads a response body, returning an error if the
+// body is larger than maxBytes.
+func readResponseBody(r io.Reader, maxBytes int64) ([]byte, error) {
+	body, err := io.ReadAll(io.LimitReader(r, maxBytes+1))
+	if err != nil {
+		return nil, fmt.Errorf("error reading response: %w", err)
+	}
+	if int64(len(body)) > maxBytes {
+		return nil, fmt.Errorf("response body exceeds maximum allowed size of %d bytes", maxBytes)
+	}
+	return body, nil
+}
+
+func get(ctx context.Context, httpClient *http.Client, fullURL string) (io.ReadCloser, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+
+	if httpClient == nil {
+		httpClient = defaultHTTPClient
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	if response.StatusCode != 200 {
+		defer response.Body.Close()
+		responseBody, _ := io.ReadAll(io.LimitReader(response.Body, maxErrorResponseBytes))
+		return nil, fmt.Errorf("Get %q: %s (%s)", fullURL, response.Status, formatResponseBody(responseBody))
+	}
+
+	return response.Body, nil
+}
+
+func getBytes(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte, error) {
+	body, err := get(ctx, httpClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+	bodyBytes, err := readResponseBody(body, maxResponseBytes)
+	if err != nil {
+		return nil, fmt.Errorf("Get %q: %w", fullURL, err)
+	}
+	return bodyBytes, nil
+}
+
+func getJSON(ctx context.Context, httpClient *http.Client, fullURL string, response any) error {
+	responseBytes, err := getBytes(ctx, httpClient, fullURL)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(responseBytes, response); err != nil {
+		return fmt.Errorf("Get %q: error parsing response JSON: %w", fullURL, err)
+	}
+	return nil
+}
+
+func getRoots(ctx context.Context, httpClient *http.Client, logURL *url.URL) ([][]byte, error) {
+	fullURL := logURL.JoinPath("/ct/v1/get-roots").String()
+	var parsedResponse struct {
+		Certificates [][]byte `json:"certificates"`
+	}
+	if err := getJSON(ctx, httpClient, fullURL, &parsedResponse); err != nil {
+		return nil, err
+	}
+	return parsedResponse.Certificates, nil
+}
+
+func addChainOrPreChain(ctx context.Context, httpClient *http.Client, logURL *url.URL, isPreChain bool, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	var endpoint string
+	if isPreChain {
+		endpoint = "/ct/v1/add-pre-chain"
+	} else {
+		endpoint = "/ct/v1/add-chain"
+	}
+	fullURL := logURL.JoinPath(endpoint).String()
+
+	requestBody, err := json.Marshal(struct {
+		Chain [][]byte `json:"chain"`
+	}{
+		Chain: chain,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(requestBody))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+	request.Header.Set("Content-Type", "application/json")
+
+	if httpClient == nil {
+		httpClient = defaultHTTPClient
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	responseBody, err := readResponseBody(response.Body, maxResponseBytes)
+	if err != nil {
+		return nil, fmt.Errorf("Post %q: %w", fullURL, err)
+	}
+
+	if response.StatusCode != 200 {
+		return nil, fmt.Errorf("Post %q: %s (%s)", fullURL, response.Status, formatResponseBody(responseBody))
+	}
+
+	sct := new(cttypes.SignedCertificateTimestamp)
+	if err := json.Unmarshal(responseBody, sct); err != nil {
+		return nil, fmt.Errorf("Post %q: error parsing response JSON: %w", fullURL, err)
+	}
+
+	return sct, nil
+}
+
+func formatResponseBody(body []byte) string {
+	const maxLen = 200
+	body = bytes.TrimSpace(body)
+	if len(body) > maxLen {
+		return strconv.QuoteToASCII(string(body[:maxLen])) + "..."
+	} else {
+		return strconv.QuoteToASCII(string(body))
+	}
+}

third_party/certspotter/ctclient/client_test.go

@@ -0,0 +1,77 @@
+// Copyright (C) 2026 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestReadResponseBody(t *testing.T) {
+	// under the limit
+	if body, err := readResponseBody(strings.NewReader("hello"), 10); err != nil {
+		t.Errorf("unexpected error under limit: %v", err)
+	} else if string(body) != "hello" {
+		t.Errorf("got %q, want %q", body, "hello")
+	}
+
+	// exactly at the limit
+	if body, err := readResponseBody(strings.NewReader("0123456789"), 10); err != nil {
+		t.Errorf("unexpected error at exact limit: %v", err)
+	} else if len(body) != 10 {
+		t.Errorf("got %d bytes, want 10", len(body))
+	}
+
+	// one byte over the limit
+	if _, err := readResponseBody(strings.NewReader("0123456789X"), 10); err == nil {
+		t.Errorf("expected error for oversize body, got nil")
+	}
+}
+
+// Confirm the cap is actually wired into getBytes(): a body larger than the limit is
+// rejected with an error instead of being buffered in full.
+func TestGetEnforcesResponseLimit(t *testing.T) {
+	const bodyLen = maxResponseBytes + 1024
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		buf := make([]byte, 32*1024)
+		for written := 0; written < bodyLen; written += len(buf) {
+			if _, err := w.Write(buf); err != nil {
+				return
+			}
+		}
+	}))
+	defer server.Close()
+
+	if _, err := getBytes(context.Background(), server.Client(), server.URL); err == nil {
+		t.Errorf("getBytes() accepted an over-limit response body; expected an error")
+	} else if !strings.Contains(err.Error(), "maximum allowed size") {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+// A normal-sized response still round-trips correctly.
+func TestGetReadsNormalBody(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "tile-data")
+	}))
+	defer server.Close()
+
+	body, err := getBytes(context.Background(), server.Client(), server.URL)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if string(body) != "tile-data" {
+		t.Errorf("got %q, want %q", body, "tile-data")
+	}
+}

third_party/certspotter/ctclient/log.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type WritableLog interface {
+	AddChain(context.Context, [][]byte) (*cttypes.SignedCertificateTimestamp, error)
+	AddPreChain(context.Context, [][]byte) (*cttypes.SignedCertificateTimestamp, error)
+	GetRoots(context.Context) ([][]byte, error)
+}
+
+type Log interface {
+	GetSTH(context.Context) (*cttypes.SignedTreeHead, string, error)
+	GetRoots(context.Context) ([][]byte, error)
+	GetEntries(ctx context.Context, startInclusive, endInclusive uint64) ([]Entry, error)
+	ReconstructTree(context.Context, *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error)
+}
+
+// IssuerGetter represents a source of issuer certificates.
+//
+// If a [Log] also implements IssuerGetter, then it is mandatory to provide
+// an IssuerGetter when using [Entry]s returned by the [Log].  The IssuerGetter
+// may be the Log itself, or your own implementation which retrieves issuers
+// from a different source, such as a cache.
+//
+// If a Log doesn't implement IssuerGetter, then you may pass a nil IssuerGetter
+// when using the Log's Entrys.
+type IssuerGetter interface {
+	GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error)
+}
+
+type Entry interface {
+	LeafInput() []byte
+
+	// Returns error from IssuerGetter, otherwise infallible
+	ExtraData(context.Context, IssuerGetter) ([]byte, error)
+
+	// Returns an error if this is not a well-formed precert entry
+	Precertificate() (cttypes.ASN1Cert, error)
+
+	// Returns an error if this is not a well-formed x509 or precert entry
+	ChainFingerprints() ([][32]byte, error)
+
+	// Returns an error if this is not a well-formed x509 or precert entry, or if IssuerGetter failed
+	GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error)
+}

third_party/certspotter/ctclient/rfc6962.go

@@ -0,0 +1,272 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"slices"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type RFC6962Log struct {
+	URL        *url.URL
+	HTTPClient *http.Client // nil to use default client
+}
+
+type RFC6962LogEntry struct {
+	Leaf_input []byte `json:"leaf_input"`
+	Extra_data []byte `json:"extra_data"`
+}
+
+func (ctlog *RFC6962Log) AddChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.URL, false, chain)
+}
+
+func (ctlog *RFC6962Log) AddPreChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.URL, true, chain)
+}
+
+func (ctlog *RFC6962Log) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-sth").String()
+	sth := new(cttypes.SignedTreeHead)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, sth); err != nil {
+		return nil, fullURL, err
+	}
+	return sth, fullURL, nil
+}
+
+func (ctlog *RFC6962Log) GetRoots(ctx context.Context) ([][]byte, error) {
+	return getRoots(ctx, ctlog.HTTPClient, ctlog.URL)
+}
+
+func (ctlog *RFC6962Log) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]RFC6962LogEntry, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-entries").String()
+	fullURL += fmt.Sprintf("?start=%d&end=%d", startInclusive, endInclusive)
+
+	body, err := get(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+
+	limitedBody := io.LimitReader(body, maxResponseBytes)
+	dec := json.NewDecoder(limitedBody)
+	if tok, err := dec.Token(); err != nil {
+		return nil, fmt.Errorf("Get %q: error reading JSON object token: %w", fullURL, err)
+	} else if tok != json.Delim('{') {
+		return nil, fmt.Errorf("Get %q: not a JSON object", fullURL)
+	}
+	for {
+		if !dec.More() {
+			return nil, fmt.Errorf("Get %q: JSON object missing entries", fullURL)
+		}
+		tok, err := dec.Token()
+		if err != nil {
+			return nil, fmt.Errorf("Get %q: error reading JSON field name: %w", fullURL, err)
+		}
+		if tok == "entries" {
+			break
+		}
+		var rawMessage json.RawMessage
+		if err := dec.Decode(&rawMessage); err != nil {
+			return nil, fmt.Errorf("Get %q: error skipping unwanted JSON field: %w", fullURL, err)
+		}
+	}
+	if tok, err := dec.Token(); err != nil {
+		return nil, fmt.Errorf("Get %q: error reading JSON array token: %w", fullURL, err)
+	} else if tok != json.Delim('[') {
+		return nil, fmt.Errorf("Get %q: entries is not a JSON array", fullURL)
+	}
+
+	var entries []RFC6962LogEntry
+	for dec.More() {
+		if uint64(len(entries)) == endInclusive-startInclusive+1 {
+			return entries, fmt.Errorf("Get %q: extraneous entries returned", fullURL)
+		}
+		var entry RFC6962LogEntry
+		if err := dec.Decode(&entry); err != nil {
+			return entries, fmt.Errorf("Get %q: error reading JSON entry: %w", fullURL, err)
+		}
+		entries = append(entries, entry)
+	}
+	if tok, err := dec.Token(); err != nil {
+		return entries, fmt.Errorf("Get %q: error reading end of JSON array: %w", fullURL, err)
+	} else if tok != json.Delim(']') {
+		return entries, fmt.Errorf("Get %q: entries array not terminated", fullURL)
+	}
+	if len(entries) == 0 {
+		return entries, fmt.Errorf("Get %q: zero entries returned", fullURL)
+	}
+	io.Copy(io.Discard, limitedBody) // So the HTTP connection can be reused
+	return entries, nil
+}
+
+func (ctlog *RFC6962Log) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) {
+	nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive)
+	var entries []Entry
+	if nativeEntries != nil {
+		entries = make([]Entry, len(nativeEntries))
+		for i := range nativeEntries {
+			entries[i] = &nativeEntries[i]
+		}
+	}
+	return entries, err
+}
+
+type entryAndProofResponse struct {
+	LeafInput []byte            `json:"leaf_input"`
+	ExtraData []byte            `json:"extra_data"`
+	AuditPath []merkletree.Hash `json:"audit_path"`
+}
+
+func (ctlog *RFC6962Log) getEntryAndProof(ctx context.Context, leafIndex uint64, treeSize uint64) (*entryAndProofResponse, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-entry-and-proof").String()
+	fullURL += fmt.Sprintf("?leaf_index=%d&tree_size=%d", leafIndex, treeSize)
+	response := new(entryAndProofResponse)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+type proofResponse struct {
+	LeafIndex uint64            `json:"leaf_index"`
+	AuditPath []merkletree.Hash `json:"audit_path"`
+}
+
+func (ctlog *RFC6962Log) getProofByHash(ctx context.Context, hash *merkletree.Hash, treeSize uint64) (*proofResponse, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-proof-by-hash").String()
+	fullURL += fmt.Sprintf("?hash=%s&tree_size=%d", url.QueryEscape(hash.Base64String()), treeSize)
+	response := new(proofResponse)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (ctlog *RFC6962Log) reconstructTree(ctx context.Context, treeSize uint64) (*merkletree.CollapsedTree, error) {
+	if treeSize == 0 {
+		return new(merkletree.CollapsedTree), nil
+	}
+	if entryAndProof, err := ctlog.getEntryAndProof(ctx, treeSize-1, treeSize); err == nil {
+		tree := new(merkletree.CollapsedTree)
+		slices.Reverse(entryAndProof.AuditPath)
+		if err := tree.Init(entryAndProof.AuditPath, treeSize-1); err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for entry %d to STH %d: %w", treeSize-1, treeSize, err)
+		}
+		tree.Add(merkletree.HashLeaf(entryAndProof.LeafInput))
+		return tree, nil
+	}
+	entries, err := ctlog.getEntries(ctx, treeSize-1, treeSize-1)
+	if err != nil {
+		return nil, err
+	}
+	leafHash := merkletree.HashLeaf(entries[0].Leaf_input)
+	tree := new(merkletree.CollapsedTree)
+	if treeSize > 1 {
+		response, err := ctlog.getProofByHash(ctx, &leafHash, treeSize)
+		if err != nil {
+			return nil, err
+		}
+		if response.LeafIndex != treeSize-1 {
+			// This can happen if the leaf hash is present in the tree in more than one place. Unfortunately, we can't reconstruct when tree if this happens. Fortunately, this is really unlikely, and most logs support get-entry-and-proof anyways.
+			return nil, fmt.Errorf("unable to reconstruct tree because leaf hash %s is present in tree at more than one index (need proof for index %d but get-proof-by-hash returned proof for index %d)", leafHash.Base64String(), treeSize-1, response.LeafIndex)
+		}
+		slices.Reverse(response.AuditPath)
+		if err := tree.Init(response.AuditPath, treeSize-1); err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for hash %s to STH %d: %w", leafHash.Base64String(), treeSize, err)
+		}
+	}
+	tree.Add(leafHash)
+	return tree, nil
+}
+
+func (ctlog *RFC6962Log) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	tree, err := ctlog.reconstructTree(ctx, sth.TreeSize)
+	if err != nil {
+		return nil, err
+	}
+
+	if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash {
+		return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (entry *RFC6962LogEntry) isX509() bool {
+	return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 0
+}
+
+func (entry *RFC6962LogEntry) isPrecert() bool {
+	return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 1
+}
+
+func (entry *RFC6962LogEntry) LeafInput() []byte {
+	return entry.Leaf_input
+}
+
+func (entry *RFC6962LogEntry) ExtraData(context.Context, IssuerGetter) ([]byte, error) {
+	return entry.Extra_data, nil
+}
+
+func (entry *RFC6962LogEntry) Precertificate() (cttypes.ASN1Cert, error) {
+	if !entry.isPrecert() {
+		return nil, fmt.Errorf("not a precertificate entry")
+	}
+	extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing extra_data: %w", err)
+	}
+	return extraData.PreCertificate, nil
+}
+
+func (entry *RFC6962LogEntry) ChainFingerprints() ([][32]byte, error) {
+	chain, err := entry.parseChain()
+	if err != nil {
+		return nil, err
+	}
+	fingerprints := make([][32]byte, len(chain))
+	for i := range chain {
+		fingerprints[i] = sha256.Sum256(chain[i])
+	}
+	return fingerprints, nil
+}
+
+func (entry *RFC6962LogEntry) GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error) {
+	return entry.parseChain()
+}
+
+func (entry *RFC6962LogEntry) parseChain() (cttypes.ASN1CertChain, error) {
+	switch {
+	case entry.isX509():
+		extraData, err := cttypes.ParseExtraDataForX509Entry(entry.Extra_data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing extra_data for X509 entry: %w", err)
+		}
+		return extraData, nil
+	case entry.isPrecert():
+		extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing extra_data for precert entry: %w", err)
+		}
+		return extraData.PrecertificateChain, nil
+	default:
+		return nil, fmt.Errorf("unknown entry type")
+	}
+}

third_party/certspotter/ctclient/static.go

@@ -0,0 +1,401 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+	"strconv"
+	"sync"
+
+	"golang.org/x/crypto/cryptobyte"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+const (
+	staticTileHeight = 8
+	StaticTileWidth  = 1 << staticTileHeight
+)
+
+func staticSubtreeSize(level uint64) uint64 { return 1 << (level * staticTileHeight) }
+
+type StaticLog struct {
+	SubmissionURL *url.URL
+	MonitoringURL *url.URL
+	ID            cttypes.LogID
+	HTTPClient    *http.Client // nil to use default client
+}
+
+type StaticLogEntry struct {
+	timestampedEntry []byte
+	precertificate   []byte // nil iff x509 entry; non-nil iff precert entry
+	chain            [][32]byte
+}
+
+func (ctlog *StaticLog) AddChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.SubmissionURL, false, chain)
+}
+
+func (ctlog *StaticLog) AddPreChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.SubmissionURL, true, chain)
+}
+
+func (ctlog *StaticLog) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) {
+	fullURL := ctlog.MonitoringURL.JoinPath("/checkpoint").String()
+	responseBody, err := getBytes(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, fullURL, err
+	}
+	sth, err := cttypes.ParseCheckpoint(responseBody, ctlog.ID)
+	if err != nil {
+		return nil, fullURL, err
+	}
+	return sth, fullURL, nil
+}
+
+func (ctlog *StaticLog) GetRoots(ctx context.Context) ([][]byte, error) {
+	return getRoots(ctx, ctlog.HTTPClient, ctlog.SubmissionURL)
+}
+
+func (ctlog *StaticLog) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]StaticLogEntry, error) {
+	var (
+		tile       = startInclusive / StaticTileWidth
+		skip       = startInclusive % StaticTileWidth
+		tileWidth  = min(StaticTileWidth, endInclusive+1-tile*StaticTileWidth)
+		numEntries = tileWidth - skip
+	)
+
+	data, err := ctlog.getDataTile(ctx, tile, tileWidth)
+	if err != nil {
+		return nil, err
+	}
+	var skippedEntry StaticLogEntry
+	for i := range skip {
+		if rest, err := skippedEntry.parse(data); err != nil {
+			return nil, fmt.Errorf("error parsing skipped entry %d in tile %d: %w", i, tile, err)
+		} else {
+			data = rest
+		}
+	}
+	entries := make([]StaticLogEntry, numEntries)
+	for i := range numEntries {
+		if rest, err := entries[i].parse(data); err != nil {
+			return nil, fmt.Errorf("error parsing entry %d in tile %d: %w", skip+i, tile, err)
+		} else {
+			data = rest
+		}
+	}
+	return entries, nil
+}
+
+func (ctlog *StaticLog) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) {
+	nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive)
+	if err != nil {
+		return nil, err
+	}
+	entries := make([]Entry, len(nativeEntries))
+	for i := range nativeEntries {
+		entries[i] = &nativeEntries[i]
+	}
+	return entries, nil
+}
+
+func (ctlog *StaticLog) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	type job struct {
+		level  uint64
+		offset uint64
+		width  uint64
+	}
+	var jobs []job
+	for level, size := uint64(0), sth.TreeSize; size > 0; level++ {
+		fullTiles := size / StaticTileWidth
+		remainder := size % StaticTileWidth
+		size = fullTiles
+		if remainder > 0 {
+			jobs = append(jobs, job{
+				level:  level,
+				offset: fullTiles,
+				width:  remainder,
+			})
+		}
+	}
+	trees := make([]*merkletree.CollapsedTree, len(jobs))
+	errs := make([]error, len(jobs))
+
+	var wg sync.WaitGroup
+	for i, job := range jobs {
+		wg.Add(1)
+		go func() {
+			trees[i], errs[i] = ctlog.getTileCollapsedTree(ctx, job.level, job.offset, job.width)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	if err := errors.Join(errs...); err != nil {
+		return nil, err
+	}
+
+	tree := new(merkletree.CollapsedTree)
+	for _, subtree := range slices.Backward(trees) {
+		if err := tree.Append(*subtree); err != nil {
+			panic(err)
+		}
+	}
+
+	if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash {
+		return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (ctlog *StaticLog) getDataTile(ctx context.Context, tile uint64, width uint64) ([]byte, error) {
+	if width == 0 || width > StaticTileWidth {
+		panic("width is out of range")
+	}
+	var partialErr error
+	if width < StaticTileWidth {
+		fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, width)).String()
+		if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+			partialErr = err
+		} else {
+			return data, nil
+		}
+	}
+
+	fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, 0)).String()
+	if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+		if partialErr != nil {
+			return nil, partialErr
+		} else {
+			return nil, err
+		}
+	} else {
+		return data, nil
+	}
+}
+
+// returned slice is numHashes*merkletree.HashLen bytes long
+func (ctlog *StaticLog) getTile(ctx context.Context, level uint64, tile uint64, numHashes uint64) ([]byte, error) {
+	if numHashes == 0 || numHashes > StaticTileWidth {
+		panic("numHashes is out of range")
+	}
+
+	var partialErr error
+	if numHashes < StaticTileWidth {
+		fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, numHashes)).String()
+		if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+			partialErr = err
+		} else if expectedLen := merkletree.HashLen * int(numHashes); len(data) != expectedLen {
+			return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen)
+		} else {
+			return data, nil
+		}
+	}
+
+	fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, 0)).String()
+	if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+		if partialErr != nil {
+			return nil, partialErr
+		} else {
+			return nil, err
+		}
+	} else if expectedLen := merkletree.HashLen * StaticTileWidth; len(data) != expectedLen {
+		return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen)
+	} else {
+		desiredLen := merkletree.HashLen * int(numHashes)
+		return data[:desiredLen], nil
+	}
+}
+
+func (ctlog *StaticLog) getTileCollapsedTree(ctx context.Context, level uint64, tile uint64, numHashes uint64) (*merkletree.CollapsedTree, error) {
+	data, err := ctlog.getTile(ctx, level, tile, numHashes)
+	if err != nil {
+		return nil, err
+	}
+	subtreeSize := staticSubtreeSize(level)
+	offset := staticSubtreeSize(level+1) * tile
+
+	tree := new(merkletree.CollapsedTree)
+	if err := tree.InitSubtree(offset, nil, 0); err != nil {
+		panic(err)
+	}
+	for i := uint64(0); i < numHashes; i++ {
+		hash := (merkletree.Hash)(data[i*merkletree.HashLen : (i+1)*merkletree.HashLen])
+		var subtree merkletree.CollapsedTree
+		if err := subtree.InitSubtree(offset+i*subtreeSize, []merkletree.Hash{hash}, subtreeSize); err != nil {
+			panic(err)
+		}
+		if err := tree.Append(subtree); err != nil {
+			panic(err)
+		}
+	}
+	return tree, nil
+}
+
+func (ctlog *StaticLog) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	fullURL := ctlog.MonitoringURL.JoinPath("/issuer/" + hex.EncodeToString(fingerprint[:])).String()
+	data, err := getBytes(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	if gotFingerprint := sha256.Sum256(data); gotFingerprint != *fingerprint {
+		return nil, fmt.Errorf("%s returned incorrect data with fingerprint %x", fullURL, gotFingerprint[:])
+	}
+	return data, nil
+}
+
+func (entry *StaticLogEntry) parse(input []byte) ([]byte, error) {
+	var skipped cryptobyte.String
+	str := cryptobyte.String(input)
+
+	// TimestampedEntry.timestamp
+	if !str.Skip(8) {
+		return nil, fmt.Errorf("error reading timestamp")
+	}
+	// TimestampedEntry.entry_type
+	var entryType uint16
+	if !str.ReadUint16(&entryType) {
+		return nil, fmt.Errorf("error reading entry type")
+	}
+	// TimestampedEntry.signed_entry
+	if entryType == 0 {
+		if !str.ReadUint24LengthPrefixed(&skipped) {
+			return nil, fmt.Errorf("error reading certificate")
+		}
+	} else if entryType == 1 {
+		if !str.Skip(32) {
+			return nil, fmt.Errorf("error reading issuer_key_hash")
+		}
+		if !str.ReadUint24LengthPrefixed(&skipped) {
+			return nil, fmt.Errorf("error reading tbs_certificate")
+		}
+	} else {
+		return nil, fmt.Errorf("invalid entry type %d", entryType)
+	}
+
+	// TimestampedEntry.extensions
+	if !str.ReadUint16LengthPrefixed(&skipped) {
+		return nil, fmt.Errorf("error reading extensions")
+	}
+
+	timestampedEntryLen := len(input) - len(str)
+	entry.timestampedEntry = input[:timestampedEntryLen]
+
+	// precertificate
+	if entryType == 1 {
+		var precertificate cryptobyte.String
+		if !str.ReadUint24LengthPrefixed(&precertificate) {
+			return nil, fmt.Errorf("error reading precertificate")
+		}
+		entry.precertificate = precertificate
+	} else {
+		entry.precertificate = nil
+	}
+
+	// certificate_chain
+	var chainBytes cryptobyte.String
+	if !str.ReadUint16LengthPrefixed(&chainBytes) {
+		return nil, fmt.Errorf("error reading certificate_chain")
+	}
+	entry.chain = make([][32]byte, 0, len(chainBytes)/32)
+	for !chainBytes.Empty() {
+		var fingerprint [32]byte
+		if !chainBytes.CopyBytes(fingerprint[:]) {
+			return nil, fmt.Errorf("error reading fingerprint in certificate_chain")
+		}
+		entry.chain = append(entry.chain, fingerprint)
+	}
+
+	return str, nil
+}
+
+func (entry *StaticLogEntry) LeafInput() []byte {
+	return append([]byte{0, 0}, entry.timestampedEntry...)
+}
+
+func (entry *StaticLogEntry) ExtraData(ctx context.Context, issuerGetter IssuerGetter) ([]byte, error) {
+	b := cryptobyte.NewBuilder(nil)
+	if entry.precertificate != nil {
+		b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(entry.precertificate)
+		})
+	}
+	b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+		for _, fingerprint := range entry.chain {
+			b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+				cert, err := issuerGetter.GetIssuer(ctx, &fingerprint)
+				if err != nil {
+					panic(cryptobyte.BuildError{Err: fmt.Errorf("error getting issuer %x: %w", fingerprint, err)})
+				}
+				b.AddBytes(cert)
+			})
+		}
+	})
+	return b.Bytes()
+}
+
+func (entry *StaticLogEntry) Precertificate() (cttypes.ASN1Cert, error) {
+	if entry.precertificate == nil {
+		return nil, fmt.Errorf("not a precertificate entry")
+	}
+	return entry.precertificate, nil
+}
+
+func (entry *StaticLogEntry) ChainFingerprints() ([][32]byte, error) {
+	return entry.chain, nil
+}
+
+func (entry *StaticLogEntry) GetChain(ctx context.Context, issuerGetter IssuerGetter) (cttypes.ASN1CertChain, error) {
+	var (
+		chain = make(cttypes.ASN1CertChain, len(entry.chain))
+		errs  = make([]error, len(entry.chain))
+	)
+	var wg sync.WaitGroup
+	for i, fingerprint := range entry.chain {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			chain[i], errs[i] = issuerGetter.GetIssuer(ctx, &fingerprint)
+		}()
+	}
+	wg.Wait()
+	if err := errors.Join(errs...); err != nil {
+		return nil, err
+	}
+	return chain, nil
+}
+
+func formatTilePath(level string, tile uint64, partial uint64) string {
+	path := "tile/" + level + "/" + formatTileIndex(tile)
+	if partial != 0 {
+		path += fmt.Sprintf(".p/%d", partial)
+	}
+	return path
+}
+
+func formatTileIndex(tile uint64) string {
+	const base = 1000
+	str := fmt.Sprintf("%03d", tile%base)
+	for tile >= base {
+		tile = tile / base
+		str = fmt.Sprintf("x%03d/%s", tile%base, str)
+	}
+	return str
+}

third_party/certspotter/ctclient/static_test.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"testing"
+)
+
+func TestFormatTileIndex(t *testing.T) {
+	tests := []struct {
+		in  uint64
+		out string
+	}{
+		{0, "000"},
+		{1, "001"},
+		{12, "012"},
+		{105, "105"},
+		{1000, "x001/000"},
+		{1050, "x001/050"},
+		{52123, "x052/123"},
+		{999001, "x999/001"},
+		{1999001, "x001/x999/001"},
+		{15999001, "x015/x999/001"},
+	}
+	for i, test := range tests {
+		result := formatTileIndex(test.in)
+		if result != test.out {
+			t.Errorf("#%d: formatTileIndex(%q) = %q, want %q", i, test.in, result, test.out)
+		}
+	}
+}

third_party/certspotter/ctcrypto/key.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctcrypto
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+
+	"software.sslmate.com/src/certspotter/tlstypes"
+)
+
+type PublicKey []byte
+
+func (key PublicKey) Verify(input SignatureInput, signature tlstypes.DigitallySigned) error {
+	parsedKey, err := x509.ParsePKIXPublicKey(key)
+	if err != nil {
+		return fmt.Errorf("error parsing log key: %w", err)
+	}
+	switch key := parsedKey.(type) {
+	case *rsa.PublicKey:
+		if signature.Algorithm.Signature != tlstypes.RSA {
+			return fmt.Errorf("log key is RSA but this is not an RSA signature")
+		}
+		if signature.Algorithm.Hash != tlstypes.SHA256 {
+			return fmt.Errorf("unsupported hash algorithm %v (only SHA-256 is allowed in CT)", signature.Algorithm.Hash)
+		}
+		if rsa.VerifyPKCS1v15((*rsa.PublicKey)(key), crypto.SHA256, input[:], signature.Signature) != nil {
+			return fmt.Errorf("RSA signature is incorrect")
+		}
+		return nil
+
+	case *ecdsa.PublicKey:
+		if signature.Algorithm.Signature != tlstypes.ECDSA {
+			return fmt.Errorf("log key is ECDSA but this is not an ECDSA signature")
+		}
+		if signature.Algorithm.Hash != tlstypes.SHA256 {
+			return fmt.Errorf("unsupported hash algorithm %v (only SHA-256 is allowed in CT)", signature.Algorithm.Hash)
+		}
+		if !ecdsa.VerifyASN1((*ecdsa.PublicKey)(key), input[:], signature.Signature) {
+			return fmt.Errorf("ECDSA signature is incorrect")
+		}
+
+	default:
+		return fmt.Errorf("unsupported public key type %T (CT only allows RSA and ECDSA)", key)
+	}
+	return nil
+}
+
+func (key PublicKey) MarshalBinary() ([]byte, error) {
+	return bytes.Clone(key), nil
+}
+
+func (key *PublicKey) UnmarshalBinary(data []byte) error {
+	*key = bytes.Clone(data)
+	return nil
+}

third_party/certspotter/ctcrypto/signatures.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctcrypto
+
+import (
+	"crypto/sha256"
+	"golang.org/x/crypto/cryptobyte"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type SignatureInput [32]byte
+
+func MakeSignatureInput(message []byte) SignatureInput {
+	return sha256.Sum256(message)
+}
+
+func SignatureInputForPrecertSCT(sct *cttypes.SignedCertificateTimestamp, precert cttypes.PreCert) SignatureInput {
+	var builder cryptobyte.Builder
+	builder.AddValue(sct.SCTVersion)
+	builder.AddValue(cttypes.CertificateTimestampSignatureType)
+	builder.AddUint64(sct.Timestamp)
+	builder.AddValue(cttypes.PrecertEntryType)
+	builder.AddValue(&precert)
+	builder.AddValue(sct.Extensions)
+	return MakeSignatureInput(builder.BytesOrPanic())
+}
+
+func SignatureInputForCertSCT(sct *cttypes.SignedCertificateTimestamp, cert cttypes.ASN1Cert) SignatureInput {
+	var builder cryptobyte.Builder
+	builder.AddValue(sct.SCTVersion)
+	builder.AddValue(cttypes.CertificateTimestampSignatureType)
+	builder.AddUint64(sct.Timestamp)
+	builder.AddValue(cttypes.X509EntryType)
+	builder.AddValue(cert)
+	builder.AddValue(sct.Extensions)
+	return MakeSignatureInput(builder.BytesOrPanic())
+}
+
+func SignatureInputForSTH(sth *cttypes.SignedTreeHead) SignatureInput {
+	var builder cryptobyte.Builder
+	builder.AddValue(cttypes.V1)
+	builder.AddValue(cttypes.TreeHashSignatureType)
+	builder.AddUint64(sth.Timestamp)
+	builder.AddUint64(sth.TreeSize)
+	builder.AddBytes(sth.RootHash[:])
+	return MakeSignatureInput(builder.BytesOrPanic())
+}

third_party/certspotter/cttypes/certs.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type TBSCertificate []byte
+
+type ASN1Cert []byte
+
+type ASN1CertChain []ASN1Cert
+
+// Corresponds to the PreCert structure in RFC 6962.  PreCert is a misnomer because this is really a TBSCertificate, not a precertificate.
+type PreCert struct {
+	IssuerKeyHash  [32]byte
+	TBSCertificate TBSCertificate
+}
+
+type PrecertChainEntry struct {
+	PreCertificate      ASN1Cert
+	PrecertificateChain ASN1CertChain
+}
+
+func (v *TBSCertificate) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint24LengthPrefixed((*cryptobyte.String)(v))
+}
+func (v TBSCertificate) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint24LengthPrefixed(addBytesFunc(v))
+	return nil
+}
+
+func (v *ASN1Cert) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint24LengthPrefixed((*cryptobyte.String)(v))
+}
+func (v ASN1Cert) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint24LengthPrefixed(addBytesFunc(v))
+	return nil
+}
+
+func (v *ASN1CertChain) Unmarshal(s *cryptobyte.String) bool {
+	chainBytes := new(cryptobyte.String)
+	if !s.ReadUint24LengthPrefixed(chainBytes) {
+		return false
+	}
+	*v = []ASN1Cert{}
+	for !chainBytes.Empty() {
+		var cert ASN1Cert
+		if !cert.Unmarshal(chainBytes) {
+			return false
+		}
+		*v = append(*v, cert)
+	}
+	return true
+}
+func (v ASN1CertChain) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+		for _, cert := range v {
+			b.AddValue(cert)
+		}
+	})
+	return nil
+}
+
+func (precert *PreCert) Unmarshal(s *cryptobyte.String) error {
+	if !s.CopyBytes(precert.IssuerKeyHash[:]) {
+		return fmt.Errorf("error reading PreCert issuer_key_hash")
+	}
+	if !precert.TBSCertificate.Unmarshal(s) {
+		return fmt.Errorf("error reading PreCert tbs_certificate")
+	}
+	return nil
+}
+func (v *PreCert) Marshal(b *cryptobyte.Builder) error {
+	b.AddBytes(v.IssuerKeyHash[:])
+	b.AddValue(v.TBSCertificate)
+	return nil
+}
+
+func (entry *PrecertChainEntry) Unmarshal(s *cryptobyte.String) error {
+	if !entry.PreCertificate.Unmarshal(s) {
+		return fmt.Errorf("error reading PrecertChainEntry pre_certificate")
+	}
+	if !entry.PrecertificateChain.Unmarshal(s) {
+		return fmt.Errorf("error reading PrecertChainEntry preeertificate_chain")
+	}
+	return nil
+}
+
+func ParseExtraDataForX509Entry(extraData []byte) (ASN1CertChain, error) {
+	str := cryptobyte.String(extraData)
+	var chain ASN1CertChain
+	if !chain.Unmarshal(&str) {
+		return nil, fmt.Errorf("error reading ASN.1Cert chain")
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after ASN.1Cert chain")
+	}
+	return chain, nil
+}
+
+func ParseExtraDataForPrecertEntry(extraData []byte) (*PrecertChainEntry, error) {
+	str := cryptobyte.String(extraData)
+	entry := new(PrecertChainEntry)
+	if err := entry.Unmarshal(&str); err != nil {
+		return nil, err
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after PrecertChainEntry")
+	}
+	return entry, nil
+}

third_party/certspotter/cttypes/checkpoint.go

@@ -0,0 +1,127 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/merkletree"
+	"software.sslmate.com/src/certspotter/tlstypes"
+)
+
+func chompLine(input []byte) (string, []byte, bool) {
+	newline := bytes.IndexByte(input, '\n')
+	if newline == -1 {
+		return "", nil, false
+	}
+	return string(input[:newline]), input[newline+1:], true
+}
+
+func makeCheckpointKeyID(origin string, logID LogID) [4]byte {
+	h := sha256.New()
+	h.Write([]byte(origin))
+	h.Write([]byte{'\n', 0x05})
+	h.Write(logID[:])
+
+	var digest [sha256.Size]byte
+	h.Sum(digest[:0])
+	return [4]byte(digest[:4])
+}
+
+func ParseCheckpoint(input []byte, logID LogID) (*SignedTreeHead, error) {
+	// origin
+	origin, input, _ := chompLine(input)
+
+	// tree size
+	sizeLine, input, _ := chompLine(input)
+	treeSize, err := strconv.ParseUint(sizeLine, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("malformed tree size: %w", err)
+	}
+
+	// root hash
+	hashLine, input, _ := chompLine(input)
+	rootHash, err := base64.StdEncoding.DecodeString(hashLine)
+	if err != nil {
+		return nil, fmt.Errorf("malformed root hash: %w", err)
+	}
+	if len(rootHash) != merkletree.HashLen {
+		return nil, fmt.Errorf("root hash has wrong length (should be %d bytes long, not %d)", merkletree.HashLen, len(rootHash))
+	}
+
+	// 0 or more non-empty extension lines (ignored)
+	for {
+		line, rest, ok := chompLine(input)
+		if !ok {
+			return nil, errors.New("signed note ended prematurely")
+		}
+		input = rest
+		if len(line) == 0 {
+			break
+		}
+	}
+
+	// signature lines
+	signaturePrefix := "\u2014 " + origin + " "
+	keyID := makeCheckpointKeyID(origin, logID)
+	for {
+		signatureLine, rest, ok := chompLine(input)
+		if !ok {
+			return nil, errors.New("signed note is missing signature from the log")
+		}
+		input = rest
+		if !strings.HasPrefix(signatureLine, signaturePrefix) {
+			continue
+		}
+		signatureBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(signatureLine, signaturePrefix))
+		if err != nil {
+			return nil, fmt.Errorf("malformed signature: %w", err)
+		}
+		if !bytes.HasPrefix(signatureBytes, keyID[:]) {
+			continue
+		}
+		if len(signatureBytes) < 12 {
+			return nil, errors.New("malformed signature: too short")
+		}
+		timestamp := binary.BigEndian.Uint64(signatureBytes[4:12])
+		signature, err := tlstypes.ParseDigitallySigned(signatureBytes[12:])
+		if err != nil {
+			return nil, fmt.Errorf("malformed signature: %w", err)
+		}
+		return &SignedTreeHead{
+			TreeSize:  treeSize,
+			Timestamp: timestamp,
+			RootHash:  (merkletree.Hash)(rootHash),
+			Signature: *signature,
+		}, nil
+	}
+}
+
+func FormatCheckpoint(sth *SignedTreeHead, origin string, logID LogID) []byte {
+	keyID := makeCheckpointKeyID(origin, logID)
+	signature := keyID[:]
+	signature = binary.BigEndian.AppendUint64(signature, sth.Timestamp)
+	signature = append(signature, sth.Signature.Bytes()...)
+
+	buf := new(bytes.Buffer)
+	fmt.Fprintln(buf, origin)
+	fmt.Fprintln(buf, sth.TreeSize)
+	fmt.Fprintln(buf, sth.RootHash.Base64String())
+	fmt.Fprintln(buf)
+	fmt.Fprintf(buf, "\u2014 %s %s\n", origin, base64.StdEncoding.EncodeToString(signature))
+	return buf.Bytes()
+}

third_party/certspotter/cttypes/checkpoint_test.go

@@ -0,0 +1,113 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"bytes"
+	"software.sslmate.com/src/certspotter/tlstypes"
+	"testing"
+)
+
+func TestParseCheckpoint(t *testing.T) {
+	logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f}
+	expected := SignedTreeHead{
+		TreeSize:  820495916,
+		Timestamp: 1719511711300,
+		RootHash:  [...]byte{0xc0, 0x3a, 0x6b, 0xe9, 0xf1, 0x1d, 0xc9, 0xcc, 0x20, 0x89, 0xe4, 0x45, 0xa7, 0x3f, 0x61, 0x41, 0xee, 0x82, 0xe8, 0x0d, 0x2d, 0x83, 0xa2, 0xb6, 0x65, 0x53, 0xd4, 0x96, 0xd7, 0x1e, 0xd2, 0x12},
+		Signature: tlstypes.DigitallySigned{
+			Algorithm: tlstypes.SignatureAndHashAlgorithm{
+				Hash:      tlstypes.SHA256,
+				Signature: tlstypes.ECDSA,
+			},
+			Signature: []byte{0x30, 0x46, 0x02, 0x21, 0x00, 0xd0, 0x0d, 0x31, 0x91, 0x50, 0x80, 0x62, 0xfa, 0xb0, 0xf9, 0xf7, 0x63, 0x61, 0x78, 0x95, 0x2b, 0x9c, 0x19, 0x22, 0x3a, 0x1d, 0x08, 0xc5, 0x68, 0x0e, 0xd0, 0x8b, 0x3b, 0x79, 0x18, 0x88, 0x86, 0x02, 0x21, 0x00, 0xd5, 0x74, 0xae, 0x8c, 0x1d, 0x1d, 0x7a, 0x4e, 0x80, 0x6c, 0x36, 0x46, 0x81, 0xb4, 0x7c, 0x91, 0x78, 0xc0, 0x3f, 0xdc, 0xc0, 0xab, 0xa5, 0x90, 0x40, 0x8d, 0x0e, 0xf6, 0x2c, 0x83, 0xa9, 0x34},
+		},
+	}
+
+	sthStrings := []string{
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\nanother extension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n\u2014 someoneelse.example c2lnbmF0dXJlCg==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 someoneelse.example c2lnbmF0dXJlCg==\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\nanother extension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2c2lnbmF0dXJlCg==\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+	}
+
+	for i, str := range sthStrings {
+		parsed, err := ParseCheckpoint([]byte(str), logID)
+		if err != nil {
+			t.Errorf("%d: Unexpected error: %s", i, err)
+			return
+		}
+		if parsed.TreeSize != expected.TreeSize {
+			t.Errorf("%d: wrong tree size", i)
+			return
+		}
+		if parsed.Timestamp != expected.Timestamp {
+			t.Errorf("%d: wrong timestamp", i)
+			return
+		}
+		if !bytes.Equal(parsed.RootHash[:], expected.RootHash[:]) {
+			t.Errorf("%d: wrong root hash", i)
+			return
+		}
+		if !(parsed.Signature.Algorithm == expected.Signature.Algorithm && bytes.Equal(parsed.Signature.Signature, expected.Signature.Signature)) {
+			t.Errorf("%d: wrong signature", i)
+			return
+		}
+	}
+}
+
+func TestParseCheckpointFailure(t *testing.T) {
+	logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f}
+	sthStrings := []string{
+		"",
+		"\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 oak.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 bm90YXNpZ25hdHVyZQo=\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 notbase64!\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916!\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEd!ycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n- sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+	}
+
+	for i, str := range sthStrings {
+		_, err := ParseCheckpoint([]byte(str), logID)
+		if err == nil {
+			t.Errorf("%d: Unexpected success", i)
+			return
+		}
+	}
+}
+
+func TestFormatCheckpoint(t *testing.T) {
+	origin := "sycamore.ct.letsencrypt.org/2024h2"
+	logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f}
+	sth := SignedTreeHead{
+		TreeSize:  820495916,
+		Timestamp: 1719511711300,
+		RootHash:  [...]byte{0xc0, 0x3a, 0x6b, 0xe9, 0xf1, 0x1d, 0xc9, 0xcc, 0x20, 0x89, 0xe4, 0x45, 0xa7, 0x3f, 0x61, 0x41, 0xee, 0x82, 0xe8, 0x0d, 0x2d, 0x83, 0xa2, 0xb6, 0x65, 0x53, 0xd4, 0x96, 0xd7, 0x1e, 0xd2, 0x12},
+		Signature: tlstypes.DigitallySigned{
+			Algorithm: tlstypes.SignatureAndHashAlgorithm{
+				Hash:      tlstypes.SHA256,
+				Signature: tlstypes.ECDSA,
+			},
+			Signature: []byte{0x30, 0x46, 0x02, 0x21, 0x00, 0xd0, 0x0d, 0x31, 0x91, 0x50, 0x80, 0x62, 0xfa, 0xb0, 0xf9, 0xf7, 0x63, 0x61, 0x78, 0x95, 0x2b, 0x9c, 0x19, 0x22, 0x3a, 0x1d, 0x08, 0xc5, 0x68, 0x0e, 0xd0, 0x8b, 0x3b, 0x79, 0x18, 0x88, 0x86, 0x02, 0x21, 0x00, 0xd5, 0x74, 0xae, 0x8c, 0x1d, 0x1d, 0x7a, 0x4e, 0x80, 0x6c, 0x36, 0x46, 0x81, 0xb4, 0x7c, 0x91, 0x78, 0xc0, 0x3f, 0xdc, 0xc0, 0xab, 0xa5, 0x90, 0x40, 0x8d, 0x0e, 0xf6, 0x2c, 0x83, 0xa9, 0x34},
+		},
+	}
+
+	expected := "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n"
+
+	got := FormatCheckpoint(&sth, origin, logID)
+	if string(got) != expected {
+		t.Errorf("expected %q, got %q", expected, got)
+	}
+}

third_party/certspotter/cttypes/helpers.go

@@ -0,0 +1,20 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+func addBytesFunc(v []byte) cryptobyte.BuilderContinuation {
+	return func(b *cryptobyte.Builder) {
+		b.AddBytes(v)
+	}
+}

third_party/certspotter/cttypes/logid.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type LogID [32]byte
+
+func (id LogID) Compare(other LogID) int {
+	return bytes.Compare(id[:], other[:])
+}
+
+func (v *LogID) Unmarshal(s *cryptobyte.String) bool {
+	return s.CopyBytes((*v)[:])
+}
+func (v LogID) Marshal(b *cryptobyte.Builder) error {
+	b.AddBytes(v[:])
+	return nil
+}
+
+func (id *LogID) UnmarshalBinary(bytes []byte) error {
+	if len(bytes) != len(*id) {
+		return fmt.Errorf("LogID has wrong length (should be %d, not %d)", len(*id), len(bytes))
+	}
+	*id = (LogID)(bytes)
+	return nil
+}
+func (id LogID) MarshalBinary() ([]byte, error) {
+	return id[:], nil
+}
+
+func (id *LogID) UnmarshalText(textData []byte) error {
+	if len(textData) != 44 {
+		return fmt.Errorf("LogID has wrong length (should be %d, not %d)", 44, len(textData))
+	}
+	var bytes [33]byte
+	if n, err := base64.StdEncoding.Decode(bytes[:], textData); err != nil {
+		return fmt.Errorf("LogID contains invalid base64: %w", err)
+	} else if n != 32 {
+		return fmt.Errorf("LogID has wrong length (should be %d bytes, not %d)", 32, n)
+	}
+	copy(id[:], bytes[:])
+	return nil
+}
+func (id LogID) MarshalText() ([]byte, error) {
+	encodedBytes := make([]byte, 44)
+	base64.StdEncoding.Encode(encodedBytes, id[:])
+	return encodedBytes, nil
+}
+
+func (id LogID) Base64String() string {
+	return base64.StdEncoding.EncodeToString(id[:])
+}
+
+func (id LogID) Base64URLString() string {
+	return base64.RawURLEncoding.EncodeToString(id[:])
+}

third_party/certspotter/cttypes/merkleleaf.go

@@ -0,0 +1,213 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type MerkleLeafType uint8
+
+const (
+	TimestampedEntryType MerkleLeafType = 0
+)
+
+type LogEntryType uint16
+
+const (
+	X509EntryType    LogEntryType = 0
+	PrecertEntryType LogEntryType = 1
+)
+
+type CTExtensions []byte
+
+type MerkleTreeLeaf struct {
+	Version          Version
+	LeafType         MerkleLeafType
+	TimestampedEntry *TimestampedEntry
+}
+
+type TimestampedEntry struct {
+	Timestamp           uint64
+	EntryType           LogEntryType
+	SignedEntryASN1Cert *ASN1Cert
+	SignedEntryPreCert  *PreCert
+	Extensions          CTExtensions
+}
+
+func (v *MerkleLeafType) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}
+func (v MerkleLeafType) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}
+
+func (v *LogEntryType) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint16((*uint16)(v))
+}
+func (v LogEntryType) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint16(uint16(v))
+	return nil
+}
+
+func (v *CTExtensions) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint16LengthPrefixed((*cryptobyte.String)(v))
+}
+func (v CTExtensions) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint16LengthPrefixed(addBytesFunc(v))
+	return nil
+}
+
+func (leaf *MerkleTreeLeaf) Unmarshal(s *cryptobyte.String) error {
+	if !leaf.Version.Unmarshal(s) {
+		return fmt.Errorf("error reading MerkleTreeLeaf version")
+	}
+	if leaf.Version != V1 {
+		return fmt.Errorf("unsupported Version 0x%02x", leaf.Version)
+	}
+	if !leaf.LeafType.Unmarshal(s) {
+		return fmt.Errorf("error reading MerkleTreeLeaf leaf_type")
+	}
+	switch leaf.LeafType {
+	case TimestampedEntryType:
+		leaf.TimestampedEntry = new(TimestampedEntry)
+		if err := leaf.TimestampedEntry.Unmarshal(s); err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unrecognized MerkleLeafType 0x%02x", leaf.LeafType)
+	}
+	return nil
+}
+func (v *MerkleTreeLeaf) Marshal(b *cryptobyte.Builder) error {
+	b.AddValue(v.Version)
+	b.AddValue(v.LeafType)
+	switch v.LeafType {
+	case TimestampedEntryType:
+		b.AddValue(v.TimestampedEntry)
+	}
+	return nil
+}
+func (v *MerkleTreeLeaf) Bytes() ([]byte, error) {
+	var builder cryptobyte.Builder
+	builder.AddValue(v)
+	return builder.Bytes()
+}
+func (v *MerkleTreeLeaf) Hash() merkletree.Hash {
+	var builder cryptobyte.Builder
+	builder.AddValue(v)
+	return merkletree.HashLeaf(builder.BytesOrPanic())
+}
+
+func (entry *TimestampedEntry) Unmarshal(s *cryptobyte.String) error {
+	if !s.ReadUint64(&entry.Timestamp) {
+		return fmt.Errorf("error reading TimestampedEntry timestamp")
+	}
+	if !entry.EntryType.Unmarshal(s) {
+		return fmt.Errorf("error reading TimestampedEntry entry_type")
+	}
+	switch entry.EntryType {
+	case X509EntryType:
+		entry.SignedEntryASN1Cert = new(ASN1Cert)
+		if !entry.SignedEntryASN1Cert.Unmarshal(s) {
+			return fmt.Errorf("error reading TimestampedEntry signed_entry ASN.1Cert")
+		}
+	case PrecertEntryType:
+		entry.SignedEntryPreCert = new(PreCert)
+		if err := entry.SignedEntryPreCert.Unmarshal(s); err != nil {
+			return fmt.Errorf("error reading TimestampedEntryType signed_entry: %w", err)
+		}
+	default:
+		return fmt.Errorf("unrecognized TimestampedEntryType 0x%02x", entry.EntryType)
+	}
+	if !entry.Extensions.Unmarshal(s) {
+		return fmt.Errorf("error reading TimestampedEntry extensions")
+	}
+	return nil
+}
+func (v *TimestampedEntry) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint64(v.Timestamp)
+	b.AddValue(v.EntryType)
+	switch v.EntryType {
+	case X509EntryType:
+		b.AddValue(v.SignedEntryASN1Cert)
+	case PrecertEntryType:
+		b.AddValue(v.SignedEntryPreCert)
+	}
+	b.AddValue(v.Extensions)
+	return nil
+}
+
+func ParseLeafInput(leafInput []byte) (*MerkleTreeLeaf, error) {
+	str := cryptobyte.String(leafInput)
+	leaf := new(MerkleTreeLeaf)
+	if err := leaf.Unmarshal(&str); err != nil {
+		return nil, err
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after MerkleTreeLeaf")
+	}
+	return leaf, nil
+}
+
+func MerkleTreeLeafForCert(timestamp uint64, extensions []byte, cert ASN1Cert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  V1,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:           timestamp,
+			EntryType:           X509EntryType,
+			SignedEntryASN1Cert: &cert,
+			Extensions:          extensions,
+		},
+	}
+}
+
+func MerkleTreeLeafForCertSCT(sct *SignedCertificateTimestamp, cert ASN1Cert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  sct.SCTVersion,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:           sct.Timestamp,
+			EntryType:           X509EntryType,
+			SignedEntryASN1Cert: &cert,
+			Extensions:          sct.Extensions,
+		},
+	}
+}
+
+func MerkleTreeLeafForPrecert(timestamp uint64, extensions []byte, precert PreCert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  V1,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:          timestamp,
+			EntryType:          PrecertEntryType,
+			SignedEntryPreCert: &precert,
+			Extensions:         extensions,
+		},
+	}
+}
+
+func MerkleTreeLeafForPrecertSCT(sct *SignedCertificateTimestamp, precert PreCert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  sct.SCTVersion,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:          sct.Timestamp,
+			EntryType:          PrecertEntryType,
+			SignedEntryPreCert: &precert,
+			Extensions:         sct.Extensions,
+		},
+	}
+}

third_party/certspotter/cttypes/sct.go

@@ -0,0 +1,73 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+	"software.sslmate.com/src/certspotter/tlstypes"
+)
+
+type SignedCertificateTimestamp struct {
+	SCTVersion Version                  `json:"sct_version"`
+	ID         LogID                    `json:"id"`
+	Timestamp  uint64                   `json:"timestamp"`
+	Extensions CTExtensions             `json:"extensions"`
+	Signature  tlstypes.DigitallySigned `json:"signature"`
+}
+
+func (sct *SignedCertificateTimestamp) Bytes() ([]byte, error) {
+	var builder cryptobyte.Builder
+	builder.AddValue(sct)
+	return builder.Bytes()
+}
+
+func (sct *SignedCertificateTimestamp) Marshal(b *cryptobyte.Builder) error {
+	b.AddValue(sct.SCTVersion)
+	b.AddValue(sct.ID)
+	b.AddUint64(sct.Timestamp)
+	b.AddValue(sct.Extensions)
+	b.AddValue(sct.Signature)
+	return nil
+}
+
+func (sct *SignedCertificateTimestamp) Unmarshal(s *cryptobyte.String) error {
+	if !sct.SCTVersion.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT version")
+	}
+	if sct.SCTVersion != V1 {
+		return fmt.Errorf("unsupported SCT version 0x%02x", sct.SCTVersion)
+	}
+	if !sct.ID.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT id")
+	}
+	if !s.ReadUint64(&sct.Timestamp) {
+		return fmt.Errorf("error reading SCT timestamp")
+	}
+	if !sct.Extensions.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT extensions")
+	}
+	if !sct.Signature.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT signature")
+	}
+	return nil
+}
+
+func ParseSignedCertificateTimestamp(data []byte) (*SignedCertificateTimestamp, error) {
+	str := cryptobyte.String(data)
+	sct := new(SignedCertificateTimestamp)
+	if err := sct.Unmarshal(&str); err != nil {
+		return nil, err
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after SignedCertificateTimestamp")
+	}
+	return sct, nil
+}

third_party/certspotter/cttypes/signatures.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type SignatureType uint8
+
+const (
+	CertificateTimestampSignatureType SignatureType = 0
+	TreeHashSignatureType             SignatureType = 1
+)
+
+func (v *SignatureType) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}
+func (v SignatureType) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}

third_party/certspotter/cttypes/sth.go

@@ -0,0 +1,37 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"software.sslmate.com/src/certspotter/merkletree"
+	"software.sslmate.com/src/certspotter/tlstypes"
+	"time"
+)
+
+type SignedTreeHead struct {
+	TreeSize  uint64                   `json:"tree_size"`
+	Timestamp uint64                   `json:"timestamp"`
+	RootHash  merkletree.Hash          `json:"sha256_root_hash"`
+	Signature tlstypes.DigitallySigned `json:"tree_head_signature"`
+}
+
+type GossipedSignedTreeHead struct {
+	SignedTreeHead
+	STHVersion Version `json:"sth_version"`
+	LogID      LogID   `json:"log_id"`
+}
+
+func (sth *SignedTreeHead) TimestampTime() time.Time {
+	return time.UnixMilli(int64(sth.Timestamp))
+}
+
+func (sth *SignedTreeHead) Same(other *SignedTreeHead) bool {
+	return sth.TreeSize == other.TreeSize && sth.Timestamp == other.Timestamp && sth.RootHash == other.RootHash
+}

third_party/certspotter/cttypes/version.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type Version uint8
+
+const (
+	V1 Version = 0
+)
+
+func (v Version) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}
+
+func (v *Version) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}

third_party/certspotter/go.mod

@@ -0,0 +1,14 @@
+module software.sslmate.com/src/certspotter
+
+go 1.25.0
+
+require (
+	golang.org/x/crypto v0.52.0
+	golang.org/x/net v0.55.0
+	golang.org/x/sync v0.20.0
+	golang.org/x/time v0.15.0
+)
+
+require golang.org/x/text v0.37.0 // indirect
+
+retract v0.19.0 // Contains serious bugs.

third_party/certspotter/go.sum

@@ -0,0 +1,22 @@
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
+golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=

third_party/certspotter/helpers.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"math/big"
+)
+
+type CertInfo struct {
+	TBS *TBSCertificate
+
+	Subject                RDNSequence
+	SubjectParseError      error
+	Issuer                 RDNSequence
+	IssuerParseError       error
+	SANs                   []SubjectAltName
+	SANsParseError         error
+	SerialNumber           *big.Int
+	SerialNumberParseError error
+	Validity               *CertValidity
+	ValidityParseError     error
+	IsCA                   *bool
+	IsCAParseError         error
+	IsPreCert              bool
+}
+
+func MakeCertInfoFromTBS(tbs *TBSCertificate) *CertInfo {
+	info := &CertInfo{TBS: tbs}
+
+	info.Subject, info.SubjectParseError = tbs.ParseSubject()
+	info.Issuer, info.IssuerParseError = tbs.ParseIssuer()
+	info.SANs, info.SANsParseError = tbs.ParseSubjectAltNames()
+	info.SerialNumber, info.SerialNumberParseError = tbs.ParseSerialNumber()
+	info.Validity, info.ValidityParseError = tbs.ParseValidity()
+	info.IsCA, info.IsCAParseError = tbs.ParseBasicConstraints()
+	info.IsPreCert = len(tbs.GetExtension(oidExtensionCTPoison)) > 0
+
+	return info
+}
+
+func MakeCertInfoFromRawTBS(tbsBytes []byte) (*CertInfo, error) {
+	tbs, err := ParseTBSCertificate(tbsBytes)
+	if err != nil {
+		return nil, err
+	}
+	return MakeCertInfoFromTBS(tbs), nil
+}
+
+func MakeCertInfoFromRawCert(certBytes []byte) (*CertInfo, error) {
+	cert, err := ParseCertificate(certBytes)
+	if err != nil {
+		return nil, err
+	}
+	return MakeCertInfoFromRawTBS(cert.GetRawTBSCertificate())
+}
+
+func MatchesWildcard(dnsName string, pattern string) bool {
+	for len(pattern) > 0 {
+		if pattern[0] == '*' {
+			if len(dnsName) > 0 && dnsName[0] != '.' && MatchesWildcard(dnsName[1:], pattern) {
+				return true
+			}
+			pattern = pattern[1:]
+		} else {
+			if len(dnsName) == 0 || pattern[0] != dnsName[0] {
+				return false
+			}
+			pattern = pattern[1:]
+			dnsName = dnsName[1:]
+		}
+	}
+	return len(dnsName) == 0
+}

third_party/certspotter/helpers_test.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"testing"
+)
+
+func doWildcardTest(t *testing.T, dnsName string, wildcard string, expected bool) {
+	if MatchesWildcard(dnsName, wildcard) != expected {
+		t.Errorf("MatchesWildcard(%q, %q) != %v", dnsName, wildcard, expected)
+	}
+}
+
+func TestMatchesWildcard(t *testing.T) {
+	doWildcardTest(t, "", "", true)
+	doWildcardTest(t, "example.com", "example.com", true)
+	doWildcardTest(t, "example.org", "example.com", false)
+	doWildcardTest(t, "example.com", "", false)
+	doWildcardTest(t, "", "example.com", false)
+	doWildcardTest(t, "", "*.example.com", false)
+	doWildcardTest(t, "", "exam*ple.com", false)
+	doWildcardTest(t, "", "exam*ple.co*m", false)
+	doWildcardTest(t, "example.org", "example.com", false)
+	doWildcardTest(t, "example.org", "*.example.com", false)
+	doWildcardTest(t, "example.org", "exam*ple.com", false)
+	doWildcardTest(t, "example.org", "exam*ple.co*m", false)
+	doWildcardTest(t, "example.com", "*.example.com", false)
+	doWildcardTest(t, "www.example.com", "*.example.com", true)
+	doWildcardTest(t, "", "*", true)
+	doWildcardTest(t, "", "****", true)
+	doWildcardTest(t, "a", "****", true)
+	doWildcardTest(t, "a", "*", true)
+	doWildcardTest(t, "a", "****", true)
+	doWildcardTest(t, "abcd", "****", true)
+	doWildcardTest(t, "abcdef", "****", true)
+	doWildcardTest(t, "www-example.com", "*-example.com", true)
+	doWildcardTest(t, "www-example-www.com", "*-example-*.com", true)
+	doWildcardTest(t, "examplecom", "example*", true)
+	doWildcardTest(t, "example.com", "example*", false)
+	doWildcardTest(t, "examplea.com", "example*", false)
+}

third_party/certspotter/identifiers.go

@@ -0,0 +1,301 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"bytes"
+	"golang.org/x/net/idna"
+	"net"
+	"strings"
+	"unicode/utf8"
+)
+
+const UnparsableDNSLabelPlaceholder = "<unparsable>"
+
+/*
+const (
+	IdentifierSourceSubjectCN = iota
+	IdentifierSourceDNSName
+	IdentifierSourceIPAddr
+)
+type IdentifierSource int
+
+type UnknownIdentifier struct {
+	Source			IdentifierSource
+	Value			[]byte
+}
+*/
+
+type Identifiers struct {
+	DNSNames []string // stored as ASCII, with IDNs in Punycode
+	IPAddrs  []net.IP
+	//Unknowns		[]UnknownIdentifier
+}
+
+func NewIdentifiers() *Identifiers {
+	return &Identifiers{
+		DNSNames: []string{},
+		IPAddrs:  []net.IP{},
+		//Unknowns:	[]UnknownIdentifier{},
+	}
+}
+
+func parseIPAddrString(str string) net.IP {
+	return net.ParseIP(str)
+}
+
+func isASCIIString(value []byte) bool {
+	for _, b := range value {
+		if b > 127 {
+			return false
+		}
+	}
+	return true
+}
+func isUTF8String(value []byte) bool {
+	return utf8.Valid(value)
+}
+func latin1ToUTF8(value []byte) string {
+	runes := make([]rune, len(value))
+	for i, b := range value {
+		runes[i] = rune(b)
+	}
+	return string(runes)
+}
+
+// Make sure the DNS label doesn't have any weird characters that
+// could cause trouble during later processing.
+func isSaneDNSLabelChar(ch rune) bool {
+	return ch == '\t' || (ch >= 32 && ch <= 126)
+}
+func isSaneDNSLabel(label string) bool {
+	for _, ch := range label {
+		if !isSaneDNSLabelChar(ch) {
+			return false
+		}
+	}
+	return true
+}
+
+func trimHttpPrefixString(value string) string {
+	if strings.HasPrefix(value, "http://") {
+		return value[7:]
+	} else if strings.HasPrefix(value, "https://") {
+		return value[8:]
+	} else {
+		return value
+	}
+}
+
+func trimHttpPrefixBytes(value []byte) []byte {
+	if bytes.HasPrefix(value, []byte("http://")) {
+		return value[7:]
+	} else if bytes.HasPrefix(value, []byte("https://")) {
+		return value[8:]
+	} else {
+		return value
+	}
+}
+
+func trimTrailingDots(value string) string {
+	length := len(value)
+	for length > 0 && value[length-1] == '.' {
+		length--
+	}
+	return value[0:length]
+}
+
+// Try to canonicalize/sanitize the DNS name:
+//  1. Trim leading and trailing whitespace
+//  2. Trim trailing dots
+//  3. Convert to lower case
+//  4. Replace totally nonsensical labels (e.g. having non-printable characters) with a placeholder
+func sanitizeDNSName(value string) string {
+	value = strings.ToLower(trimTrailingDots(strings.TrimSpace(value)))
+	labels := strings.Split(value, ".")
+	for i, label := range labels {
+		if !isSaneDNSLabel(label) {
+			labels[i] = UnparsableDNSLabelPlaceholder
+		}
+	}
+	return strings.Join(labels, ".")
+}
+
+// Like sanitizeDNSName, but labels that are Unicode are converted to Punycode.
+func sanitizeUnicodeDNSName(value string) string {
+	value = strings.ToLower(trimTrailingDots(strings.TrimSpace(value)))
+	labels := strings.Split(value, ".")
+	for i, label := range labels {
+		if asciiLabel, err := idna.ToASCII(label); err == nil && isSaneDNSLabel(asciiLabel) {
+			labels[i] = asciiLabel
+		} else {
+			labels[i] = UnparsableDNSLabelPlaceholder
+		}
+	}
+	return strings.Join(labels, ".")
+}
+
+func (ids *Identifiers) appendDNSName(dnsName string) {
+	if dnsName != "" && !ids.hasDNSName(dnsName) {
+		ids.DNSNames = append(ids.DNSNames, dnsName)
+	}
+}
+func (ids *Identifiers) appendIPAddress(ipaddr net.IP) {
+	if !ids.hasIPAddress(ipaddr) {
+		ids.IPAddrs = append(ids.IPAddrs, ipaddr)
+	}
+}
+
+func (ids *Identifiers) hasDNSName(target string) bool {
+	for _, value := range ids.DNSNames {
+		if value == target {
+			return true
+		}
+	}
+	return false
+}
+func (ids *Identifiers) hasIPAddress(target net.IP) bool {
+	for _, value := range ids.IPAddrs {
+		if value.Equal(target) {
+			return true
+		}
+	}
+	return false
+}
+
+func (ids *Identifiers) addDnsSANfinal(value []byte) {
+	if ipaddr := parseIPAddrString(string(value)); ipaddr != nil {
+		// Stupid CAs put IP addresses in DNS SANs because stupid Microsoft
+		// used to not support IP address SANs.  Since there's no way for an IP
+		// address to also be a valid DNS name, just treat it like an IP address
+		// and not try to process it as a DNS name.
+		ids.appendIPAddress(ipaddr)
+	} else if isASCIIString(value) {
+		ids.appendDNSName(sanitizeDNSName(string(value)))
+	} else {
+		// DNS SANs are supposed to be IA5Strings (i.e. ASCII) but CAs can't follow
+		// simple rules.  Unfortunately, we have no idea what the encoding really is
+		// in this case, so interpret it as both UTF-8 (if it's valid UTF-8)
+		// and Latin-1.
+		if isUTF8String(value) {
+			ids.appendDNSName(sanitizeUnicodeDNSName(string(value)))
+		}
+		ids.appendDNSName(sanitizeUnicodeDNSName(latin1ToUTF8(value)))
+	}
+}
+
+func (ids *Identifiers) addDnsSANnonull(value []byte) {
+	if slashIndex := bytes.IndexByte(value, '/'); slashIndex != -1 {
+		// If the value contains a slash, then this might be a URL,
+		// so process the part of the value up to the first slash,
+		// which should be the domain.  Even though no client should
+		// ever successfully validate such a DNS name, the domain owner
+		// might still want to know about it.
+		ids.addDnsSANfinal(value[0:slashIndex])
+	}
+	ids.addDnsSANfinal(value)
+}
+
+func (ids *Identifiers) AddDnsSAN(value []byte) {
+	// Trim http:// and https:// prefixes, which are all too common in the wild,
+	// so http://example.com becomes just example.com.  Even though clients
+	// should never successfully validate a DNS name like http://example.com,
+	// the owner of example.com might still want to know about it.
+	value = trimHttpPrefixBytes(value)
+
+	if nullIndex := bytes.IndexByte(value, 0); nullIndex != -1 {
+		// If the value contains a null byte, process the part of
+		// the value up to the first null byte in addition to the
+		// complete value, in case this certificate is an attempt to
+		// fake out validators that only compare up to the first null.
+		ids.addDnsSANnonull(value[0:nullIndex])
+	}
+	ids.addDnsSANnonull(value)
+}
+
+func (ids *Identifiers) addCNfinal(value string) {
+	if ipaddr := parseIPAddrString(value); ipaddr != nil {
+		ids.appendIPAddress(ipaddr)
+	} else if !strings.ContainsRune(value, ' ') {
+		// If the CN contains a space it's clearly not a DNS name, so ignore it.
+		ids.appendDNSName(sanitizeUnicodeDNSName(value))
+	}
+}
+
+func (ids *Identifiers) addCNnonull(value string) {
+	if slashIndex := strings.IndexRune(value, '/'); slashIndex != -1 {
+		// If the value contains a slash, then this might be a URL,
+		// so process the part of the value up to the first slash,
+		// which should be the domain.  Even though no client should
+		// ever successfully validate such a DNS name, the domain owner
+		// might still want to know about it.
+		ids.addCNfinal(value[0:slashIndex])
+	}
+	ids.addCNfinal(value)
+}
+
+func (ids *Identifiers) AddCN(value string) {
+	// Trim http:// and https:// prefixes, which are all too common in the wild,
+	// so http://example.com becomes just example.com.  Even though clients
+	// should never successfully validate a DNS name like http://example.com,
+	// the owner of example.com might still want to know about it.
+	value = trimHttpPrefixString(value)
+
+	if nullIndex := strings.IndexRune(value, 0); nullIndex != -1 {
+		// If the value contains a null byte, process the part of
+		// the value up to the first null byte in addition to the
+		// complete value, in case this certificate is an attempt to
+		// fake out validators that only compare up to the first null.
+		ids.addCNnonull(value[0:nullIndex])
+	}
+	ids.addCNnonull(value)
+}
+
+func (ids *Identifiers) AddIPAddress(value net.IP) {
+	ids.appendIPAddress(value)
+}
+
+func (cert *CertInfo) ParseIdentifiers() (*Identifiers, error) {
+	ids := NewIdentifiers()
+
+	if cert.SubjectParseError != nil {
+		return nil, cert.SubjectParseError
+	}
+	cns, err := cert.Subject.ParseCNs()
+	if err != nil {
+		return nil, err
+	}
+	for _, cn := range cns {
+		ids.AddCN(cn)
+	}
+
+	if cert.SANsParseError != nil {
+		return nil, cert.SANsParseError
+	}
+	for _, san := range cert.SANs {
+		switch san.Type {
+		case sanDNSName:
+			ids.AddDnsSAN(san.Value)
+		case sanIPAddress:
+			if len(san.Value) == 4 || len(san.Value) == 16 {
+				ids.AddIPAddress(net.IP(san.Value))
+			}
+			// TODO: decide what to do with IP addresses with an invalid length.
+			// The two encoding errors I've observed in CT logs are:
+			//  1. encoding the IP address as a string
+			//  2. a value of 0x00000000FFFFFF00 (WTF?)
+			// IP addresses aren't a high priority so just ignore invalid ones for now.
+			// Hopefully no clients out there are dumb enough to process IP address
+			// SANs encoded as strings...
+		}
+	}
+
+	return ids, nil
+}

third_party/certspotter/loglist/helpers.go

@@ -0,0 +1,40 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+)
+
+// Return all tiled and non-tiled logs from all operators
+func (list *List) AllLogs() []*Log {
+	logs := []*Log{}
+	for operator := range list.Operators {
+		for log := range list.Operators[operator].Logs {
+			logs = append(logs, &list.Operators[operator].Logs[log])
+		}
+		for log := range list.Operators[operator].TiledLogs {
+			logs = append(logs, &list.Operators[operator].TiledLogs[log])
+		}
+	}
+	return logs
+}
+
+func (log *Log) LogIDString() string {
+	return log.LogID.Base64String()
+}
+
+func (log *Log) AcceptsExpiration(expiration time.Time) bool {
+	return log.TemporalInterval == nil || withinInterval(expiration, log.TemporalInterval.StartInclusive, log.TemporalInterval.EndExclusive)
+}
+
+func withinInterval(expiration, startInclusive, endExclusive time.Time) bool {
+	return !expiration.Before(startInclusive) && expiration.Before(endExclusive)
+}

third_party/certspotter/loglist/load.go

@@ -0,0 +1,129 @@
+// Copyright (C) 2020, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+var UserAgent = ""
+
+// maxResponseBytes bounds the size of a log list we are willing to read into memory.
+const maxResponseBytes = 16 << 20 // 16 MiB
+
+// httpClient is used for fetching log lists.
+var httpClient = &http.Client{
+	Timeout: 60 * time.Second,
+}
+
+type ModificationToken struct {
+	etag     string
+	modified time.Time
+}
+
+var ErrNotModified = errors.New("loglist has not been modified")
+
+func newModificationToken(response *http.Response) *ModificationToken {
+	token := &ModificationToken{
+		etag: response.Header.Get("ETag"),
+	}
+	if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil {
+		token.modified = t
+	}
+	return token
+}
+
+func (token *ModificationToken) setRequestHeaders(request *http.Request) {
+	if token.etag != "" {
+		request.Header.Set("If-None-Match", token.etag)
+	} else if !token.modified.IsZero() {
+		request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat))
+	}
+}
+
+func Load(ctx context.Context, urlOrFile string) (*List, error) {
+	list, _, err := LoadIfModified(ctx, urlOrFile, nil)
+	return list, err
+}
+
+func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) {
+	if strings.HasPrefix(urlOrFile, "https://") {
+		return FetchIfModified(ctx, urlOrFile, token)
+	} else {
+		list, err := ReadFile(urlOrFile)
+		return list, nil, err
+	}
+}
+
+func Fetch(ctx context.Context, url string) (*List, error) {
+	list, _, err := FetchIfModified(ctx, url, nil)
+	return list, err
+}
+
+func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+	if token != nil {
+		token.setRequestHeaders(request)
+	}
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, nil, err
+	}
+	content, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
+	response.Body.Close()
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(content) > maxResponseBytes {
+		return nil, nil, fmt.Errorf("%s: response body exceeds maximum allowed size of %d bytes", url, maxResponseBytes)
+	}
+	if token != nil && response.StatusCode == http.StatusNotModified {
+		return nil, nil, ErrNotModified
+	}
+	if response.StatusCode != 200 {
+		return nil, nil, fmt.Errorf("%s: %s", url, response.Status)
+	}
+	list, err := Unmarshal(content)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing %s: %w", url, err)
+	}
+	return list, newModificationToken(response), err
+}
+
+func ReadFile(filename string) (*List, error) {
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return Unmarshal(content)
+}
+
+func Unmarshal(jsonBytes []byte) (*List, error) {
+	list := new(List)
+	if err := json.Unmarshal(jsonBytes, list); err != nil {
+		return nil, err
+	}
+	if err := list.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid log list: %s", err)
+	}
+	return list, nil
+}

third_party/certspotter/loglist/mozilla/ctlogs.go

@@ -0,0 +1,275 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+// Package mozilla contains a parser for Mozilla's CTKnownLogs.h file
+package mozilla
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// CTLogInfo describes a certificate transparency log from Mozilla's CTKnownLogs.h
+// file.
+type CTLogInfo struct {
+	Name          string
+	State         string // Admissible or Retired
+	Protocol      string // RFC6962 or Tiled
+	Timestamp     time.Time
+	OperatorIndex int
+	Key           []byte
+}
+
+// CTLogOperatorInfo describes a CT log operator from Mozilla's CTKnownLogs.h
+// file.
+type CTLogOperatorInfo struct {
+	Name string
+	ID   int
+}
+
+// Parse reads the CTKnownLogs.h content from r and returns the parsed logs and
+// operators. Blocks enclosed by `#ifdef DEBUG` and `#endif` are ignored.
+func Parse(r io.Reader) ([]CTLogInfo, []CTLogOperatorInfo, error) {
+	scanner := bufio.NewScanner(r)
+	skip := 0
+	inLogs := false
+	inOps := false
+
+	var logs []CTLogInfo
+	var ops []CTLogOperatorInfo
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		trimmed := strings.TrimSpace(line)
+
+		switch {
+		case strings.HasPrefix(trimmed, "#ifdef DEBUG"):
+			skip++
+			continue
+		case strings.HasPrefix(trimmed, "#endif"):
+			if skip > 0 {
+				skip--
+				continue
+			}
+		}
+
+		if skip > 0 {
+			continue
+		}
+
+		if strings.HasPrefix(trimmed, "const CTLogInfo kCTLogList[]") {
+			inLogs = true
+			continue
+		}
+		if strings.HasPrefix(trimmed, "const CTLogOperatorInfo kCTLogOperatorList[]") {
+			inOps = true
+			continue
+		}
+
+		if inLogs {
+			if trimmed == "};" {
+				inLogs = false
+				continue
+			}
+			if strings.HasPrefix(trimmed, "{") {
+				log, err := readLogEntry(trimmed, scanner)
+				if err != nil {
+					return nil, nil, err
+				}
+				logs = append(logs, log)
+			}
+			continue
+		}
+
+		if inOps {
+			if trimmed == "};" {
+				inOps = false
+				continue
+			}
+			if strings.HasPrefix(trimmed, "{") {
+				op, err := readOperatorEntry(trimmed)
+				if err != nil {
+					return nil, nil, err
+				}
+				ops = append(ops, op)
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, nil, err
+	}
+
+	return logs, ops, nil
+}
+
+func readLogEntry(firstLine string, s *bufio.Scanner) (CTLogInfo, error) {
+	var log CTLogInfo
+
+	// Example first line:
+	// {"Name", CTLogState::Admissible, CTLogFormat::RFC6962,
+	firstLine = strings.TrimSpace(firstLine)
+	if !strings.HasPrefix(firstLine, "{") {
+		return log, errors.New("invalid log entry start")
+	}
+	firstLine = strings.TrimPrefix(firstLine, "{")
+	firstLine = strings.TrimSuffix(firstLine, ",")
+
+	parts := splitCSV(firstLine)
+
+	for len(parts) < 3 {
+		if !s.Scan() {
+			return log, io.ErrUnexpectedEOF
+		}
+		next := strings.TrimSpace(strings.TrimSuffix(s.Text(), ","))
+		parts = append(parts, splitCSV(next)...)
+	}
+
+	log.Name = trimQuotes(parts[0])
+	log.State = strings.TrimPrefix(strings.TrimSpace(parts[1]), "CTLogState::")
+	log.Protocol = strings.TrimPrefix(strings.TrimSpace(parts[2]), "CTLogFormat::")
+
+	// Next line: timestamp
+	if !s.Scan() {
+		return log, io.ErrUnexpectedEOF
+	}
+	tsLine := strings.TrimSpace(s.Text())
+	tsValue := strings.Split(tsLine, ",")[0]
+	ts, err := strconv.ParseInt(strings.TrimSpace(tsValue), 10, 64)
+	if err != nil {
+		return log, err
+	}
+	log.Timestamp = time.Unix(0, ts*int64(time.Millisecond))
+
+	// Next line: operator index
+	if !s.Scan() {
+		return log, io.ErrUnexpectedEOF
+	}
+	opLine := strings.TrimSpace(s.Text())
+	opValue := strings.Split(opLine, ",")[0]
+	opIndex, err := strconv.Atoi(strings.TrimSpace(opValue))
+	if err != nil {
+		return log, err
+	}
+	log.OperatorIndex = opIndex
+
+	// Key lines
+	var keyHex strings.Builder
+	for {
+		if !s.Scan() {
+			return log, io.ErrUnexpectedEOF
+		}
+		l := strings.TrimSpace(s.Text())
+		if strings.HasPrefix(l, "\"") {
+			// remove trailing comma if any
+			trimmed := strings.TrimSuffix(l, ",")
+			keyHex.WriteString(trimQuotes(trimmed))
+			if strings.HasSuffix(l, ",") {
+				// last key line
+				break
+			}
+			continue
+		}
+		return log, errors.New("unexpected line while reading key")
+	}
+
+	key, err := decodeHexEscapes(keyHex.String())
+	if err != nil {
+		return log, err
+	}
+
+	// key length line
+	if !s.Scan() {
+		return log, io.ErrUnexpectedEOF
+	}
+	lenLine := strings.TrimSpace(s.Text())
+	lenValue := strings.TrimSuffix(lenLine, "},")
+	keyLen, err := strconv.Atoi(strings.TrimSpace(lenValue))
+	if err != nil {
+		return log, err
+	}
+	if len(key) != keyLen {
+		// ignore mismatch but continue
+	}
+	log.Key = key
+	return log, nil
+}
+
+func readOperatorEntry(line string) (CTLogOperatorInfo, error) {
+	var op CTLogOperatorInfo
+	line = strings.TrimSuffix(strings.TrimSpace(line), ",")
+	if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") {
+		return op, errors.New("invalid operator entry")
+	}
+	line = strings.TrimPrefix(line, "{")
+	line = strings.TrimSuffix(line, "}")
+	parts := splitCSV(line)
+	if len(parts) != 2 {
+		return op, errors.New("invalid operator fields")
+	}
+	op.Name = trimQuotes(parts[0])
+	id, err := strconv.Atoi(strings.TrimSpace(parts[1]))
+	if err != nil {
+		return op, err
+	}
+	op.ID = id
+	return op, nil
+}
+
+func splitCSV(s string) []string {
+	var parts []string
+	var cur strings.Builder
+	inQuote := false
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if c == '"' {
+			inQuote = !inQuote
+			cur.WriteByte(c)
+			continue
+		}
+		if c == ',' && !inQuote {
+			parts = append(parts, strings.TrimSpace(cur.String()))
+			cur.Reset()
+			continue
+		}
+		cur.WriteByte(c)
+	}
+	if cur.Len() > 0 {
+		parts = append(parts, strings.TrimSpace(cur.String()))
+	}
+	return parts
+}
+
+func trimQuotes(s string) string {
+	s = strings.TrimSpace(s)
+	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
+		return s[1 : len(s)-1]
+	}
+	return s
+}
+
+func decodeHexEscapes(s string) ([]byte, error) {
+	var out []byte
+	for i := 0; i < len(s); {
+		if i+3 >= len(s) || s[i] != '\\' || s[i+1] != 'x' {
+			return nil, errors.New("invalid escape")
+		}
+		b, err := strconv.ParseUint(s[i+2:i+4], 16, 8)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, byte(b))
+		i += 4
+	}
+	return out, nil
+}

third_party/certspotter/loglist/mozilla/ctlogs_test.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package mozilla
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"testing"
+)
+
+// parseFromURL downloads the CTKnownLogs.h file from the given URL and parses it.
+func parseFromURL(url string) ([]CTLogInfo, []CTLogOperatorInfo, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, nil, errors.New(resp.Status)
+	}
+	return Parse(resp.Body)
+}
+
+func TestParseFromURL(t *testing.T) {
+	logs, ops, err := parseFromURL("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ops) == 0 {
+		t.Fatal("no operators parsed")
+	}
+	foundGoogle := false
+	foundSectigo := false
+	foundLets := false
+	for _, op := range ops {
+		if op.Name == "" {
+			t.Error("operator with empty name")
+		}
+		switch op.Name {
+		case "Google":
+			foundGoogle = true
+		case "Sectigo":
+			foundSectigo = true
+		case "Let's Encrypt":
+			foundLets = true
+		}
+	}
+	if !foundGoogle || !foundSectigo || !foundLets {
+		t.Errorf("missing expected operators: Google=%v Sectigo=%v Let's=%v", foundGoogle, foundSectigo, foundLets)
+	}
+
+	if len(logs) == 0 {
+		t.Fatal("no logs parsed")
+	}
+	foundHash := false
+	targetHash := "1219ENGn9XfCx+lf1wC/+YLJM1pl4dCzAXMXwMjFaXc="
+	for _, l := range logs {
+		if l.Name == "" {
+			t.Error("log with empty name")
+		}
+		if len(l.Key) == 0 {
+			t.Error("log with empty key")
+		}
+		if l.State != "Admissible" && l.State != "Retired" {
+			t.Errorf("unexpected state %q", l.State)
+		}
+		if l.Protocol != "RFC6962" && l.Protocol != "Tiled" {
+			t.Errorf("unexpected protocol %q", l.Protocol)
+		}
+		hash := sha256.Sum256(l.Key)
+		if base64.StdEncoding.EncodeToString(hash[:]) == targetHash {
+			foundHash = true
+		}
+	}
+	if !foundHash {
+		t.Errorf("log with key hash %s not found", targetHash)
+	}
+}

third_party/certspotter/loglist/mozilla/example_test.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package mozilla_test
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"software.sslmate.com/src/certspotter/loglist/mozilla"
+)
+
+func ExampleParse() {
+	resp, err := http.Get("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Fatal(resp.Status)
+	}
+	logs, operators, err := mozilla.Parse(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0)
+	fmt.Fprintln(tw, "Operator\tName")
+	for _, o := range operators {
+		fmt.Fprintf(tw, "%d\t%s\n", o.ID, o.Name)
+	}
+	tw.Flush()
+
+	tw = tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0)
+	fmt.Fprintln(tw, "LogID\tProtocol\tState\tTimestamp\tOperator\tName")
+	for _, l := range logs {
+		hash := sha256.Sum256(l.Key)
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%s\n",
+			base64.StdEncoding.EncodeToString(hash[:]),
+			l.Protocol, l.State, l.Timestamp.UTC().Format(time.RFC3339), l.OperatorIndex, l.Name)
+	}
+	tw.Flush()
+}

third_party/certspotter/loglist/schema.go

@@ -0,0 +1,158 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type List struct {
+	Version          string     `json:"version,omitzero"`
+	LogListTimestamp time.Time  `json:"log_list_timestamp,omitzero"` // Only present in v3 of schema
+	Operators        []Operator `json:"operators,omitzero"`
+}
+
+type Operator struct {
+	Name      string   `json:"name"`
+	Email     []string `json:"email,omitzero"`
+	Logs      []Log    `json:"logs,omitzero"`
+	TiledLogs []Log    `json:"tiled_logs,omitzero"`
+}
+
+// Log represents the log object in the "v3" JSON log list schema, as published by Chrome, Apple, and SSLMate.
+type Log struct {
+	Key              []byte        `json:"key"`
+	LogID            cttypes.LogID `json:"log_id"`
+	MMD              int           `json:"mmd"`
+	URL              string        `json:"url,omitzero"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitzero"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitzero"` // only for static-ct-api logs
+	Description      string        `json:"description"`
+	State            State         `json:"state,omitzero"`
+	LogType          LogType       `json:"log_type,omitzero"`
+	TemporalInterval *struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval,omitzero"`
+	LogSoftware *struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+	} `json:"log_software,omitzero"`
+
+	// certspotter-specific extensions
+	CertspotterDownloadSize int     `json:"certspotter_download_size,omitzero"`
+	CertspotterDownloadJobs int     `json:"certspotter_download_jobs,omitzero"`
+	CertspotterDownloadQPS  float64 `json:"certspotter_download_qps,omitzero"`
+
+	// TODO: add previous_operators
+}
+
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
+type State struct {
+	Pending *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"pending,omitzero"`
+
+	Qualified *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"qualified,omitzero"`
+
+	Usable *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"usable,omitzero"`
+
+	Readonly *struct {
+		Timestamp     time.Time `json:"timestamp"`
+		FinalTreeHead struct {
+			TreeSize       int64  `json:"tree_size"`
+			SHA256RootHash []byte `json:"sha256_root_hash"`
+		} `json:"final_tree_head"`
+	} `json:"readonly,omitzero"`
+
+	Retired *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"retired,omitzero"`
+
+	Rejected *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"rejected,omitzero"`
+}
+
+func (state *State) IsApproved() bool {
+	return state.Qualified != nil || state.Usable != nil || state.Readonly != nil
+}
+
+func (state *State) WasApprovedAt(t time.Time) bool {
+	return state.Retired != nil && t.Before(state.Retired.Timestamp)
+}
+
+type LogType string
+
+const (
+	LogTypeProd = "prod"
+	LogTypeTest = "test"
+)
+
+type LogSpec string
+
+const (
+	StaticCTAPI = "static-ct-api"
+	RFC6962     = "rfc6962"
+)
+
+// InclusionRequest represents the "v2" JSON log object, as published by log operators to request inclusion in browsers.
+type InclusionRequest struct {
+	Key          []byte        `json:"key"`
+	LogID        cttypes.LogID `json:"log_id"`
+	MMDSeconds   int           `json:"mmd_seconds"`
+	FriendlyName string        `json:"friendly_name"`
+	LogSpec      LogSpec       `json:"log_spec"`
+	Endpoint     struct {
+		URL string `json:"url"`
+	} `json:"endpoint,omitzero"`
+	SubmissionEndpoint struct {
+		URL string `json:"url"`
+	} `json:"submission_endpoint,omitzero"`
+	MonitoringEndpoint struct {
+		URL string `json:"url"`
+	} `json:"monitoring_endpoint,omitzero"`
+	TemporalInterval struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval"`
+	LogSoftware struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+	} `json:"log_software"`
+}

third_party/certspotter/loglist/validate.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+func (list *List) Validate() error {
+	for i := range list.Operators {
+		if err := list.Operators[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth operator (%s): %w", i, list.Operators[i].Name, err)
+		}
+	}
+	return nil
+}
+
+func (operator *Operator) Validate() error {
+	for i := range operator.Logs {
+		if err := operator.Logs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+		}
+	}
+	for i := range operator.TiledLogs {
+		if err := operator.TiledLogs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
+		}
+	}
+	return nil
+}
+
+func (log *Log) Validate() error {
+	realLogID := sha256.Sum256(log.Key)
+	if log.LogID != realLogID {
+		return fmt.Errorf("log ID does not match log key")
+	}
+
+	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
+		return fmt.Errorf("URL(s) not provided")
+	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
+		return fmt.Errorf("inconsistent URLs provided")
+	}
+
+	if log.MMD < 0 {
+		return fmt.Errorf("log has a negative MMD")
+	}
+
+	return nil
+}

third_party/certspotter/man/.gitignore

@@ -0,0 +1 @@
+*.8

third_party/certspotter/man/Makefile

@@ -0,0 +1,9 @@
+all: certspotter-script.8 certspotter.8 certspotter-authorize.8 certspotter-metrics.8
+
+%.8: %.md
+	lowdown -s -Tman \
+		-M title:$(basename $(notdir $@)) \
+		-M section:$(subst .,,$(suffix $@)) \
+		-M date:$(if $(SOURCE_DATE_EPOCH),$(shell date -I -u -d "@$(SOURCE_DATE_EPOCH)"),$(shell date -I -u)) \
+		-o $@ $<
+

third_party/certspotter/man/certspotter-authorize.md

@@ -0,0 +1,105 @@
+# NAME
+
+**certspotter-authorize** - Authorize certificates to suppress certspotter notifications
+
+# SYNOPSIS
+
+**certspotter-authorize** [`-printhash`] *CERTFILE*...
+
+**certspotter-authorize** [`-printhash`] `-`
+
+# DESCRIPTION
+
+**certspotter-authorize** is a utility for preemptively authorizing
+certificates so that **certspotter(8)** will not send notifications when
+those certificates are discovered in Certificate Transparency logs.
+
+This is useful for preventing false alarms when you know in advance that a
+certificate will be issued. For example, you might run **certspotter-authorize**
+immediately after receiving a certificate from your certificate authority, as
+part of your certificate issuance pipeline.
+
+**certspotter-authorize** uses the TBSCertificate hash as defined by RFC
+6962 Section 3.2 to identify certificates. This hash is the same for a
+certificate and its corresponding precertificate. This means authorizing
+the certificate will suppress notifications for the precertificate
+as well. Certificates with different serial numbers, validity periods,
+or other changes to the TBSCertificate will not be covered by the authorization
+and will trigger notifications.
+
+# ARGUMENTS
+
+**certspotter-authorize** takes paths to one or more PEM-encoded certificates
+on the command line. If `-` is specified as the sole path, it reads the certificate
+from stdin.
+
+# OPTIONS
+
+-state\_dir *PATH*
+
+:   Directory where certspotter stores state. Defaults to
+    `$CERTSPOTTER_STATE_DIR` if set, or `~/.certspotter` otherwise.
+    This should be the same directory used by **certspotter(8)**.
+
+-printhash
+
+:   Instead of authorizing the certificate, print its TBS hash (a hex-encoded SHA-256 digest) to stdout.
+
+-version
+
+:   Print version information and exit.
+
+# EXAMPLES
+
+Authorize a certificate from a file:
+
+    $ certspotter-authorize /path/to/cert.pem
+
+Authorize multiple certificates from files:
+
+    $ certspotter-authorize cert-a.pem cert-b.pem
+
+Authorize a certificate from stdin:
+
+    $ cat cert.pem | certspotter-authorize -
+
+Authorize a certificate in a custom state directory:
+
+    $ certspotter-authorize -state_dir /var/lib/certspotter cert.pem
+
+Print the TBS hash without authorizing:
+
+    $ certspotter-authorize -printhash /path/to/cert.pem
+
+# OPERATION
+
+**certspotter-authorize** computes the SHA-256 hash of each
+certificate's TBSCertificate as defined by RFC 6962 Section 3.2 and
+creates a `.notified` marker file in the certspotter
+state directory. When certspotter later discovers a certificate with
+the same TBSCertificate in a CT log, it will skip sending notifications
+because the marker file is present.
+
+# ENVIRONMENT
+
+`CERTSPOTTER_STATE_DIR`
+
+:   Directory for storing state. Overridden by `-state_dir`. Defaults to
+    `~/.certspotter`.  This should be the same directory used by **certspotter(8)**.
+
+# FILES
+
+`$CERTSPOTTER_STATE_DIR/certs/XX/.HASH.notified`
+
+:   Marker files indicating that a certificate with TBS hash `HASH`
+    has been authorized. `XX` is the first two hex digits of the hash.
+    The file is empty; only its existence is checked.
+
+# EXIT STATUS
+
+**certspotter-authorize** exits with status 0 on success, 1 on error, or 2 on
+invalid usage.
+
+# SEE ALSO
+
+certspotter(8), certspotter-script(8)

third_party/certspotter/man/certspotter-metrics.md

@@ -0,0 +1,128 @@
+# NAME
+
+**certspotter-metrics** - Output Prometheus metrics about certspotter's operation
+
+# SYNOPSIS
+
+**certspotter-metrics** [`-state_dir` *PATH*]
+
+# DESCRIPTION
+
+**certspotter-metrics** reads a certspotter state directory and writes
+Prometheus metrics, in the Prometheus text exposition format, about
+certspotter's progress monitoring each Certificate Transparency log.
+Metrics are written to standard output.
+
+The intended use is to run **certspotter-metrics** periodically (e.g., from
+cron) and expose its output to Prometheus, for example with the
+prometheus-node-exporter(1) textfile collector.
+
+**certspotter-metrics** only reads the state directory; it never modifies it.
+It is safe to run while certspotter is running, and it does not require
+certspotter to be running.
+
+Metrics are output for every log for which certspotter has a state directory
+(`$STATE_DIR/logs/LOG_ID`, where `LOG_ID` is the base64url-encoded log ID).
+A log has a state directory once certspotter has started monitoring it.
+The `log_id` label on each metric is the standard base64 encoding (with
+padding) of the 32-byte log ID defined by RFC 6962. See <https://sslmate.com/app/ctlogs>
+for a list of human-friendly names for each log ID.
+
+**certspotter-metrics** is experimental, and the metrics may change in a future release.
+
+# OPTIONS
+
+-state\_dir *PATH*
+
+:   Directory where certspotter stores state. Defaults to
+    `$CERTSPOTTER_STATE_DIR` if set, or `~/.certspotter` otherwise.
+    This should be the same directory used by **certspotter(8)**.
+
+-version
+
+:   Print version information and exit.
+
+# METRICS
+
+All metrics are gauges with a `log_id` label identifying the log the
+metric pertains to.  If a metric does not pertain to a log (because
+certspotter has not stored the relevant state for the log yet), the log
+does not have a sample for that metric.
+
+certspotter\_log\_max\_sth\_size
+
+:   Tree size of the largest signed tree head (STH) certspotter has observed
+    from the log; in other words, the number of entries in the log.  Absent
+    if certspotter has not yet downloaded an STH from the log.
+
+certspotter\_log\_max\_sth\_timestamp
+
+:   Timestamp, in milliseconds since the Epoch, of the STH reported by
+    certspotter\_log\_max\_sth\_size.
+
+certspotter\_log\_download\_position
+
+:   Number of entries certspotter has downloaded from the log, including
+    entries that have not yet been verified.
+
+certspotter\_log\_verified\_position
+
+:   Number of entries certspotter has downloaded from the log and verified
+    against an STH.
+
+certspotter\_log\_verified\_sth\_timestamp
+
+:   Timestamp, in milliseconds since the Epoch, of the most recent STH
+    certspotter has verified for the log. Absent if certspotter has not
+    verified an STH for the log. The size of this STH is equal to
+    certspotter\_log\_verified\_position.
+
+certspotter\_log\_unverified\_sths
+
+:   Number of unverified STHs certspotter is currently storing for the log.
+
+certspotter\_log\_malformed\_entries
+
+:   Number of malformed entries certspotter has recorded for the log.
+    certspotter records an entry as malformed when it cannot parse the
+    entry. See **certspotter(8)** for more information.
+
+certspotter\_log\_health\_check\_failures
+
+:   Number of failed health checks certspotter has recorded for the log.
+    See **certspotter(8)** for information about health checks.
+
+# EXAMPLES
+
+crontab(5) entry that exposes metrics to the node\_exporter textfile collector
+every five minutes (writing to a temporary file and renaming it
+ensures that node\_exporter never reads a partially-written file):
+
+    */5 * * * * root certspotter-metrics > /var/lib/node_exporter/certspotter.prom.tmp && mv /var/lib/node_exporter/certspotter.prom.tmp /var/lib/node_exporter/certspotter.prom
+
+Prometheus query to determine download backlog (number entries in the
+log that haven't been downloaded yet):
+
+    certspotter_log_max_sth_size-certspotter_log_download_position
+
+# ENVIRONMENT
+
+`CERTSPOTTER_STATE_DIR`
+
+:   Directory for storing state. Overridden by `-state_dir`. Defaults to
+    `~/.certspotter`. This should be the same directory used by
+    **certspotter(8)**.
+
+`STATE_DIRECTORY`
+
+:   Directory for storing state, if set and certspotter-metrics is running
+    under systemd.
+
+# EXIT STATUS
+
+**certspotter-metrics** exits with status 0 on success, 1 on error, or 2 on
+invalid usage.
+
+# SEE ALSO
+
+certspotter(8)

third_party/certspotter/man/certspotter-script.md

@@ -0,0 +1,296 @@
+# NAME
+
+**certspotter-script** - Certificate Transparency Log Monitor (hook script)
+
+# DESCRIPTION
+
+**certspotter-script** is *any* program that is executed by **certspotter(8)**
+when it needs to notify you about an event, such as detecting a certificate for
+a domain on your watch list.
+
+Scripts are placed in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
+(`~/.certspotter/hooks.d` by default), or specified on the command line
+using the `-script` argument.
+
+# ENVIRONMENT
+
+## Event information
+
+The following environment variables are set for all types of events:
+
+`EVENT`
+
+:   One of the following values, indicating the type of event:
+
+      * `discovered_cert` - certspotter has discovered a certificate for a
+      domain on your watch list.
+
+      * `malformed_cert` - certspotter can't determine if a certificate
+      matches your watch list because the certificate or the log entry
+      is malformed.
+
+      * `error` - a problem is preventing certspotter from monitoring all
+      logs.
+
+    Additional event types may be defined in the future, so your script should
+    be able to handle unknown values.
+
+`SUMMARY`
+
+:   A short human-readable string describing the event.  This is the same string
+    used in the subject line of emails sent by certspotter.
+
+
+## Discovered certificate information
+
+The following environment variables are set for `discovered_cert` events:
+
+`WATCH_ITEM`
+
+:    The item from your watch list which matches this certificate.
+     (If more than one item matches, the first one is used.)
+
+`LOG_URI`
+
+:    The URI of the log containing the certificate.
+
+`ENTRY_INDEX`
+
+:    The index of the log entry containing the certificate.
+
+`TBS_SHA256`
+
+:    The hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2.
+     Certificates and their corresponding precertificates have the same `TBS_SHA256` value.
+     You can compute the TBS digest of a certificate using the `-printhash` option of the certspotter-authorize(8) command.
+
+`CERT_SHA256`
+
+:    The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate or precertificate.
+     The digest is computed over the ASN.1 DER encoding. 
+
+`PUBKEY_SHA256`
+
+:    The hex-encoded SHA-256 digest of the certificate's Subject Public Key Info.
+
+`CERT_FILENAME`
+
+:    Path to a file containing the PEM-encoded certificate chain.  When `-no_save` is used, this points to a temporary file that is automatically deleted after your script completes.
+
+`JSON_FILENAME`
+
+:    Path to a JSON file containing additional information about the certificate.  See below for the format of the JSON file.
+     When `-no_save` is used, this points to a temporary file that is automatically deleted after your script completes.
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing information about the certificate.  This file contains the same text that
+     certspotter uses in emails.  You should not attempt to parse this file because its format may change in the future.
+     When `-no_save` is used, this points to a temporary file that is automatically deleted after your script completes.
+
+`NOT_BEFORE`, `NOT_BEFORE_UNIXTIME`, `NOT_BEFORE_RFC3339`
+
+:    The not before time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively.  These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set.
+
+`NOT_AFTER`, `NOT_AFTER_UNIXTIME`, `NOT_AFTER_RFC3339`
+
+:    The not after (expiration) time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively.  These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set.
+
+`VALIDITY_PARSE_ERROR`
+
+:    Error parsing not before and not after, if any.  If this variable is set, then the `NOT_BEFORE` and `NOT_AFTER` family of variables are unset.
+
+`SUBJECT_DN`
+
+:    The distinguished name of the certificate's subject.  This variable may be unset if there was a parse error, in which case `SUBJECT_PARSE_ERROR` is set.
+
+`SUBJECT_PARSE_ERROR`
+
+:    Error parsing the subject, if any.  If this variable is set, then `SUBJECT_DN` is unset.
+
+`ISSUER_DN`
+
+:    The distinguished name of the certificate's issuer.  This variable may be unset if there was a parse error, in which case `ISSUER_PARSE_ERROR` is set.  Be aware that this value is controlled by the certificate authority and often contains [misleading, incorrect, or unhelpful information](https://www.agwa.name/blog/post/the_certificate_issuer_field_is_a_lie).
+
+`ISSUER_PARSE_ERROR`
+
+:    Error parsing the issuer, if any.  If this variable is set, then `ISSUER_DN` is unset.
+
+`SERIAL`
+
+:    The hex-encoded serial number of the certificate.  Prefixed with a minus (-) sign if negative.  This variable may be unset if there was a parse error, in which case `SERIAL_PARSE_ERROR` is set.  Be aware that malicious certificate authorities can issue certificates with duplicate serial numbers, so it is not safe to use the serial number to make security decisions (such as to determine if a certificate is legitimate or not).
+
+`SERIAL_PARSE_ERROR`
+
+:    Error parsing the serial number, if any.  If this variable is set, then `SERIAL` is unset.
+
+`CHAIN_ERROR`
+
+:    Error building or verifying the certificate chain, if any.  If this variable is set, then the certificate chain in `CERT_FILENAME` may be incomplete or invalid.
+
+## Malformed certificate information
+
+The following environment variables are set for `malformed_cert` events:
+
+`LOG_URI`
+
+:    The URI of the log containing the malformed certificate.
+
+`ENTRY_INDEX`
+
+:    The index of the log entry containing the malformed certificate.
+
+`LEAF_HASH`
+
+:    The base64-encoded Merkle hash of the leaf containing the malformed certificate.
+
+`PARSE_ERROR`
+
+:    A human-readable string describing why the certificate is malformed.
+
+`ENTRY_FILENAME`
+
+:    Path to a file containing the JSON log entry.  The file contains a JSON object with two fields, `leaf_input` and `extra_data`, as described in RFC 6962 Section 4.6.
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing a description of the malformed certificate.  This file contains the same text that certspotter uses in emails.
+
+## Error information
+
+The following environment variables are set for `error` events:
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing a description of the error.  This file contains the same text that certspotter uses in emails.
+
+# JSON FILE FORMAT
+
+certspotter saves a JSON file for every discovered certificate and puts the path to the file in `$JSON_FILENAME`.
+When `-no_save` is used, the JSON file is created in a temporary location and automatically deleted after your script completes.
+Otherwise, it is saved in the state directory for permanent storage. Your script can read the JSON file, such as with the jq(1) command,
+to get additional information about the certificate which isn't appropriate for environment variables.
+
+The JSON file contains an object with the following fields:
+
+`tbs_sha256`
+
+:    A string containing the hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2.
+     Certificates and their corresponding precertificates have the same `tbs_sha256` value.  You can compute the
+     TBS digest of a certificate using the `-printhash` option of the certspotter-authorize(8) command.
+
+`pubkey_sha256`
+
+:    A string containing the hex-encoded SHA-256 digest of the certificate's Subject Public Key Info.
+
+`dns_names`
+
+:    An array of strings containing the DNS names for which the
+     certificate is valid, taken from both the DNS subject alternative names
+     (SANs) and the subject common name (CN). Internationalized domain names
+     are encoded in Punycode.
+
+`ip_addresses`
+
+:    An array of strings containing the IP addresses for which the certificate is valid,
+     taken from both the IP subject alternative names (SANs) and the subject common name (CN).
+
+`not_before`
+
+:    A string containing the not before time of the certificate in RFC3339 format.
+     Null if there was an error parsing the certificate's validity.
+
+`not_after`
+
+:    A string containing the not after (expiration) time of the certificate in RFC3339 format.
+     Null if there was an error parsing the certificate's validity.
+
+Additional fields will be added in the future based on user feedback. Please open
+an issue at <https://github.com/SSLMate/certspotter> if you have a use case for another field.
+
+# DETERMINING IF A CERTIFICATE IS AUTHORIZED
+
+You may want to compare a discovered certificate against a database of legitimate certificates
+to decide whether to raise an alarm about it. Beware the following caveats:
+
+ * Since certificates may be logged to CT in the form of a precertificate, it doesn't work
+   to compare the discovered certificate's fingerprint against a list of legitimate fingerprints.
+
+ * Since malicious certificate authorities can issue certificates with duplicate serial numbers,
+   it is not secure to compare the the discovered certificate's serial number against a list of
+   legitimate serial numbers.
+
+ * Since malicious certificate authorities can put any value in a certificate's Subject Key Identifier (SKI)
+   extension, it is not secure to compare a discovered certificate's Subject Key Identifier against
+   a list of legitimate Subject Key Identifiers.
+
+ * Since malicious certificate authorities can put any value in the certificate's Issuer field,
+   it is not secure to compare a discovered certificate's Issuer against a list of authorized Issuers.
+
+To determine if a certificate is legitimate, you should do one of the following:
+
+ * Compare the discovered certificate's TBS hash (as provided in the `$TBS_SHA256` variable) against
+   a list of authorized TBS hashes. The TBS hash is computed over the DER encoding of the certificate's
+   TBSCertificate, *with any SCT extension removed*. You can use the `-printhash` option
+   of the certspotter-authorize(8) command to compute a certificate's TBS hash.
+
+ * Compare the discovered certificate's public key hash (as provided in the `$PUBKEY_SHA256` variable) against
+   a list of authorized public key hashes. The public key hash is computed over the DER encoding of the
+   certificate's Subject Public Key Info. You can use the following openssl command to compute a certificate's
+   public key hash: `openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | openssl sha256`
+
+
+# EXAMPLES
+
+Example environment variables for a `discovered_cert` event:
+
+```
+CERT_FILENAME=/home/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.pem
+CERT_SHA256=3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a
+ENTRY_INDEX=6464843
+EVENT=discovered_cert
+ISSUER_DN=C=GB, ST=Greater Manchester, L=Salford, O=Sectigo Limited, CN=Sectigo RSA Domain Validation Secure Server CA
+JSON_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.v1.json
+LOG_URI=https://ct.cloudflare.com/logs/nimbus2024/
+NOT_AFTER='2024-01-26 03:47:26 +0000 UTC'
+NOT_AFTER_RFC3339=2024-01-26T03:47:26Z
+NOT_AFTER_UNIXTIME=1706240846
+NOT_BEFORE='2023-01-31 03:47:26 +0000 UTC'
+NOT_BEFORE_RFC3339=2023-01-31T03:47:26Z
+NOT_BEFORE_UNIXTIME=1675136846
+PUBKEY_SHA256=33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8
+SERIAL=c170fbf3bf27481e5c351a4db6f2dc5f
+SUBJECT_DN=CN=sslmate.com
+SUMMARY='certificate discovered for .sslmate.com'
+TBS_SHA256=2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8
+TEXT_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.txt
+WATCH_ITEM=.sslmate.com
+```
+
+Example JSON file for a discovered certificate:
+
+```
+{
+  "dns_names": [
+    "sslmate.com",
+    "www.sslmate.com"
+  ],
+  "ip_addresses": [],
+  "not_after": "2024-01-26T03:47:26Z",
+  "not_before": "2023-01-31T03:47:26Z",
+  "pubkey_sha256": "33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8",
+  "tbs_sha256": "2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8"
+}
+```
+
+# SEE ALSO
+
+certspotter(8)
+
+# COPYRIGHT
+
+Copyright (c) 2016-2025 Opsmate, Inc.
+
+# BUGS
+
+Report bugs to <https://github.com/SSLMate/certspotter>.

third_party/certspotter/man/certspotter.md

@@ -0,0 +1,261 @@
+# NAME
+
+**certspotter** - Certificate Transparency Log Monitor
+
+# SYNOPSIS
+
+**certspotter** [`-start_at_end`] [`-watchlist` *FILENAME*] [`-email` *ADDRESS*] `...`
+
+# DESCRIPTION
+
+**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that
+alerts you when a SSL/TLS certificate is issued for one of your domains.
+Cert Spotter is easier to use than other open source CT monitors, since it does not require
+a database. It's also more robust, since it uses a special certificate parser
+that ensures it won't miss certificates.
+
+Cert Spotter is also available as a hosted service by SSLMate,
+<https://sslmate.com/certspotter>.
+
+You can use Cert Spotter to detect:
+
+ * Certificates issued to attackers who have compromised your DNS and
+   are redirecting your visitors to their malicious site.
+ * Certificates issued to attackers who have taken over an abandoned
+   sub-domain in order to serve malware under your name.
+ * Certificates issued to attackers who have compromised a certificate
+   authority and want to impersonate your site.
+ * Certificates issued in violation of your corporate policy
+   or outside of your centralized certificate procurement process.
+
+# OPTIONS
+
+-email *ADDRESS*
+
+:   Email address to contact when a matching certificate is discovered, or
+    an error occurs.  You can specify this option more than once to email
+    multiple addresses.  Your system must have a working sendmail(1) command.
+
+    Regardless of the `-email` option, certspotter also emails any address listed
+    in `$CERTSPOTTER_CONFIG_DIR/email_recipients` file
+    (`~/.certspotter/email_recipients` by default).  (One address per line,
+    blank lines are ignored.)  This file is read only at startup, so you
+    must restart certspotter if you change it.
+
+-healthcheck *INTERVAL*
+
+:   Perform a health check at the given interval (default: "24h") as described
+    below.  *INTERVAL* must be a decimal number followed by "h" for hours or
+    "m" for minutes.
+
+-logs *ADDRESS*
+
+:   Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor.
+    The schema for this file can be found at <https://www.gstatic.com/ct/log_list/v3/log_list_schema.json>.
+    Defaults to <https://loglist.certspotter.org/monitor.json>, which includes
+    the union of active logs recognized by Chrome and Apple.  certspotter loads the
+    log list when starting up, and periodically reloads it in case it has changed.
+
+-no\_save
+
+:   Do not save a copy of matching certificates. Note that enabling this option
+    will cause you to receive duplicate notifications, since certspotter will
+    have no way of knowing if you've been previously notified about a certificate.
+
+-script *COMMAND*
+
+:   Command to execute when a matching certificate is found or an error occurs. See
+    certspotter-script(8) for information about the interface to scripts.
+
+    Regardless of the `-script` option, certspotter also executes any executable
+    file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
+    (`~/.certspotter/hooks.d` by default).
+
+-start\_at\_end
+
+:   Start monitoring logs from the end rather than the beginning.
+
+    **WARNING**: monitoring from the beginning guarantees detection of all
+    certificates, but requires downloading hundreds of millions of
+    certificates, which takes days.
+
+-state\_dir *PATH*
+
+:   Directory for storing state. Defaults to `$CERTSPOTTER_STATE_DIR`, which is
+    "~/.certspotter" by default.
+
+-stdout
+
+:   Write matching certificates and errors to stdout.
+
+-verbose
+
+:   Print detailed information about certspotter's operation to stderr.
+
+-version
+
+:   Print version and exit.
+
+-watchlist *PATH*
+
+:   File containing DNS names to monitor, one per line.  To monitor an entire
+    domain namespace (including the domain itself and all sub-domains) prefix
+    the domain name with a dot (e.g. ".example.com").  To monitor a single DNS
+    name only, do not prefix the name with a dot.
+    
+    Defaults to `$CERTSPOTTER_CONFIG_DIR/watchlist`, which is
+    "~/.certspotter/watchlist" by default.
+    Specify `-` to read the watch list from stdin.
+    
+    certspotter reads the watch list only when starting up, so you must restart
+    certspotter if you change it.
+
+# NOTIFICATIONS
+
+When certspotter detects a certificate matching your watchlist, or encounters
+an error that is preventing it from discovering certificates, it notifies you
+as follows:
+
+* Emails any address specified by the `-email` command line flag.
+
+* Emails any address listed in the `$CERTSPOTTER_CONFIG_DIR/email_recipients`
+  file (`~/.certspotter/email_recipients` by default).  (One address per line,
+  blank lines are ignored.)  This file is read only at startup, so you
+  must restart certspotter if you change it.
+
+* Executes the script specified by the `-script` command line flag.
+
+* Executes every executable file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d`
+ directory (`~/.certspotter/hooks.d` by default).
+
+* Writes the notification to standard out if the `-stdout` flag was specified.
+
+Sending email requires a working sendmail(1) command.  For details about
+the script interface, see certspotter-script(8).
+
+# OPERATION
+
+certspotter continuously monitors all browser-recognized Certificate
+Transparency logs (both RFC6962 and static-ct-api) looking for certificates (including precertificates)
+which are valid for any domain on your watch list. When certspotter
+detects a matching certificate, it emails you, executes a script, and/or
+writes a report to standard out, as described above.
+
+certspotter also saves a copy of matching certificates in
+`$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default)
+unless you specify the `-no_save` option.
+
+When certspotter has not previously monitored a log, it can either start
+monitoring the log from the beginning, or seek to the end of the log and
+start monitoring from there. Monitoring from the beginning guarantees
+detection of all certificates, but requires downloading hundreds of
+millions of certificates, which takes days. The default behavior is to
+monitor from the beginning. To start monitoring new logs from the end,
+specify the `-start_at_end` option.
+
+If certspotter has previously monitored a log, it resumes monitoring
+the log from the previous position.  This means that if you add
+a domain to your watch list, certspotter will not detect any certificates
+that were logged prior to the addition.  To detect such certificates,
+you must delete `$CERTSPOTTER_STATE_DIR/logs`, which will cause certspotter
+to restart monitoring from the very beginning of each log (provided
+`-start_at_end` is not specified).  This will cause certspotter to download
+hundreds of millions of certificates, which takes days.  To find preexisting
+certificates, it's faster to use the Cert Spotter service
+<https://sslmate.com/certspotter>, SSLMate's Certificate Transparency Search
+API <https://sslmate.com/ct_search_api>, or a CT search engine such as
+<https://crt.sh>.
+
+# ERROR HANDLING
+
+When certspotter encounters a problem with the local system (e.g. failure
+to write a file, send an email, or execute a script), it prints a message to stderr and
+exits with a non-zero status.
+
+When certspotter encounters a problem loading the log list during startup, it
+prints a message to stderr and exits with a non-zero status.  When certspotter encounters a problem
+reloading the log list, it prints a message to stderr and continues running with the previously-loaded
+log list.  It will try reloading the log list again later.
+
+When certspotter encounters a problem contacting a log, it writes the error to a file in
+the state directory and continues running.  It will try contacting the log again later;
+most log errors are transient.  The last 7 days of errors are kept.
+
+Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the
+following health checks:
+
+ * Ensure that the log list has been successfully retrieved at least once
+   since the previous health check.
+ * Ensure that every log has been successfully contacted at least once
+   since the previous health check.
+ * Ensure that certspotter is not falling behind monitoring any logs.
+
+If any health check fails, certspotter notifies you by email, script, and/or
+standard out, as described above.  The notification includes the last several errors
+encountered when contacting the log.
+
+Health check failures should be rare, and you should take them seriously because it means
+certspotter might not detect all certificates.  It might also be an indication
+of CT log misbehavior.  Check the error files for details, and if
+you need help, file an issue at <https://github.com/SSLMate/certspotter>.
+
+# EXIT STATUS
+
+certspotter exits 0 when it receives `SIGTERM` or `SIGINT`,
+and non-zero when a serious error occurs.
+
+# ENVIRONMENT
+
+`CERTSPOTTER_STATE_DIR`
+
+:   Directory for storing state. Overridden by `-state_dir`. Defaults to
+    `~/.certspotter`.
+
+`CERTSPOTTER_CONFIG_DIR`
+
+:   Directory from which any configuration, such as the watch list, is read.
+    Defaults to `~/.certspotter`.
+
+`EMAIL`
+
+:   Email address from which to send emails. If not set, certspotter lets sendmail pick
+    the address.
+
+`HTTPS_PROXY`
+
+:   URL of proxy server for making HTTPS requests.  `http://`, `https://`, and
+    `socks5://` URLs are supported.  By default, no proxy server is used.
+
+`SENDMAIL_PATH`
+
+:   Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`.
+
+If the `SYSTEMD_EXEC_PID` environment variable equals the PID of certspotter, then certspotter assumes it is running under systemd and respects the `CONFIGURATION_DIRECTORY`, `STATE_DIRECTORY`, and `CACHE_DIRECTORY` environment variables, unless overridden by a certspotter-specific environment variable or flag.
+
+certspotter respects the `XDG_CACHE_HOME` environment variable for locating its cache directory. For historical reasons, certspotter does not respect `XDG_CONFIG_HOME` or `XDG_STATE_HOME`.
+
+# DIRECTORIES
+
+Config directory
+
+: Stores configuration, such as the watch list. The location is: (1) the `CERTSPOTTER_CONFIG_DIR` environment variable, if set, (2) the `CONFIGURATION_DIRECTORY` environment variable, if set and certspotter is running under systemd, or (3) the default location `~/.certspotter`. certspotter does not write to this directory.
+
+State directory
+
+: Stores state, such as the position of each log and a store of discovered certificates. The location is: (1) the `-state_dir` command line flag, if provided, (2) the `CERTSPOTTER_STATE_DIR` environment variable, if set, (3) the `STATE_DIRECTORY` environment variable, if set and certspotter is running under systemd, or (4) the default location `~/.certspotter`.  certspotter creates this directory if necessary.
+
+Cache directory
+
+: Stores cached data. The location is (1) the `CERTSPOTTER_CACHE_DIR` environment variable if set, (2) the `CACHE_DIRECTORY` environment, if set and certspotter is running under systemd, or (3) `$XDG_CACHE_HOME/certspotter` (which on Linux is `~/.cache/certspotter` by default).  You can delete this directory without impacting functionality, but certspotter may need to perform additional computation or network requests.
+
+# SEE ALSO
+
+certspotter-script(8)
+
+# COPYRIGHT
+
+Copyright (c) 2016-2025 Opsmate, Inc.
+
+# BUGS
+
+Report bugs to <https://github.com/SSLMate/certspotter>.

third_party/certspotter/merkletree/collapsed_tree.go

@@ -0,0 +1,182 @@
+// Copyright (C) 2022 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/bits"
+	"slices"
+)
+
+// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
+// nodes and calculating the root hash.
+type CollapsedTree struct {
+	offset uint64
+	nodes  []Hash
+	size   uint64
+}
+
+func calculateNumNodes(size uint64) int {
+	return bits.OnesCount64(size)
+}
+
+// TODO: phase out this function
+func EmptyCollapsedTree() *CollapsedTree {
+	return &CollapsedTree{nodes: []Hash{}, size: 0}
+}
+
+// TODO: phase out this function
+func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
+	tree := new(CollapsedTree)
+	if err := tree.Init(nodes, size); err != nil {
+		return nil, err
+	}
+	return tree, nil
+}
+
+func (tree CollapsedTree) Equal(other CollapsedTree) bool {
+	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
+}
+
+func (tree CollapsedTree) Clone() CollapsedTree {
+	return CollapsedTree{
+		offset: tree.offset,
+		nodes:  slices.Clone(tree.nodes),
+		size:   tree.size,
+	}
+}
+
+// Add a new leaf hash to the end of the tree.
+// Returns an error if and only if the new tree would be too large for the subtree offset.
+// Always returns a nil error if tree.Offset() == 0.
+func (tree *CollapsedTree) Add(hash Hash) error {
+	if tree.offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if tree.size+1 > maxSize {
+			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
+		}
+	}
+	tree.nodes = append(tree.nodes, hash)
+	tree.size++
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) Append(other CollapsedTree) error {
+	if tree.offset+tree.size != other.offset {
+		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
+	}
+	if tree.offset > 0 {
+		newSize := tree.size + other.size
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if newSize > maxSize {
+			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
+		}
+	}
+	if tree.size > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+		if other.size > maxSize {
+			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+		}
+	}
+
+	tree.nodes = append(tree.nodes, other.nodes...)
+	tree.size += other.size
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) collapse() {
+	numNodes := calculateNumNodes(tree.size)
+	for len(tree.nodes) > numNodes {
+		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
+		tree.nodes = tree.nodes[:len(tree.nodes)-2]
+		tree.nodes = append(tree.nodes, HashChildren(left, right))
+	}
+}
+
+func (tree CollapsedTree) CalculateRoot() Hash {
+	if len(tree.nodes) == 0 {
+		return HashNothing()
+	}
+	i := len(tree.nodes) - 1
+	hash := tree.nodes[i]
+	for i > 0 {
+		i -= 1
+		hash = HashChildren(tree.nodes[i], hash)
+	}
+	return hash
+}
+
+// Return the subtree offset (0 if this represents an entire tree)
+func (tree CollapsedTree) Offset() uint64 {
+	return tree.offset
+}
+
+// Return a non-nil slice containing the nodes.  The slice
+// must not be modified.
+func (tree CollapsedTree) Nodes() []Hash {
+	if tree.nodes == nil {
+		return []Hash{}
+	} else {
+		return tree.nodes
+	}
+}
+
+// Return the number of leaf nodes in the tree.
+func (tree CollapsedTree) Size() uint64 {
+	return tree.size
+}
+
+type collapsedTreeMessage struct {
+	Offset uint64 `json:"offset,omitzero"`
+	Nodes  []Hash `json:"nodes"` // never nil
+	Size   uint64 `json:"size"`
+}
+
+func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(collapsedTreeMessage{
+		Offset: tree.offset,
+		Nodes:  tree.Nodes(),
+		Size:   tree.size,
+	})
+}
+
+func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
+	var rawTree collapsedTreeMessage
+	if err := json.Unmarshal(b, &rawTree); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}
+
+func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
+	if len(nodes) != calculateNumNodes(size) {
+		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	}
+	tree.size = size
+	tree.nodes = nodes
+	return nil
+}
+
+func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
+	if offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(offset)
+		if size > maxSize {
+			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
+		}
+	}
+	tree.offset = offset
+	return tree.Init(nodes, size)
+}

third_party/certspotter/merkletree/fragment.go

@@ -0,0 +1,122 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+)
+
+// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
+type FragmentedCollapsedTree struct {
+	subtrees []CollapsedTree // sorted by offset
+}
+
+func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
+	return tree.Add(CollapsedTree{
+		offset: position,
+		nodes:  []Hash{hash},
+		size:   1,
+	})
+}
+
+func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
+	if subtree.size == 0 {
+		return nil
+	}
+	i := len(tree.subtrees)
+	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
+		i--
+	}
+	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
+		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
+	}
+	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
+		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
+	}
+	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
+		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
+		i++
+	}
+	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
+		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
+	}
+	return nil
+}
+
+func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
+	// TODO: try to make this linear time instead of quadratic; it should be possible since the subtrees are sorted by offset
+	for _, subtree := range other.subtrees {
+		if err := tree.Add(subtree); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
+	var prevEnd uint64
+	for i := range tree.subtrees {
+		if prevEnd != tree.subtrees[i].offset {
+			if !yield(prevEnd, tree.subtrees[i].offset) {
+				return
+			}
+		}
+		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
+	}
+	yield(prevEnd, 0)
+}
+
+func (tree FragmentedCollapsedTree) NumSubtrees() int {
+	return len(tree.subtrees)
+}
+
+func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
+	return tree.subtrees[i]
+}
+
+func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
+	if tree.subtrees == nil {
+		return []CollapsedTree{}
+	} else {
+		return tree.subtrees
+	}
+}
+
+// Return true iff the tree contains at least the first n nodes (without any gaps)
+func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
+	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
+}
+
+func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
+	for i := 1; i < len(subtrees); i++ {
+		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
+			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
+		}
+	}
+	tree.subtrees = subtrees
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(tree.Subtrees())
+}
+
+func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
+	var subtrees []CollapsedTree
+	if err := json.Unmarshal(b, &subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.Init(subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}

third_party/certspotter/merkletree/hash.go

@@ -0,0 +1,77 @@
+// Copyright (C) 2022 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+)
+
+const HashLen = 32
+
+type Hash [HashLen]byte
+
+func (h Hash) Compare(other Hash) int {
+	return bytes.Compare(h[:], other[:])
+}
+
+func (h Hash) Base64String() string {
+	return base64.StdEncoding.EncodeToString(h[:])
+}
+
+func (h Hash) MarshalJSON() ([]byte, error) {
+	return json.Marshal(h[:])
+}
+
+func (h Hash) MarshalBinary() ([]byte, error) {
+	return h[:], nil
+}
+
+func (h *Hash) UnmarshalJSON(b []byte) error {
+	var hashBytes []byte
+	if err := json.Unmarshal(b, &hashBytes); err != nil {
+		return err
+	}
+	return h.UnmarshalBinary(hashBytes)
+}
+
+func (h *Hash) UnmarshalBinary(hashBytes []byte) error {
+	if len(hashBytes) != HashLen {
+		return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes))
+	}
+	copy(h[:], hashBytes)
+	return nil
+}
+
+func HashNothing() Hash {
+	return sha256.Sum256(nil)
+}
+
+func HashLeaf(leafBytes []byte) Hash {
+	var hash Hash
+	hasher := sha256.New()
+	hasher.Write([]byte{0x00})
+	hasher.Write(leafBytes)
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+func HashChildren(left Hash, right Hash) Hash {
+	var hash Hash
+	hasher := sha256.New()
+	hasher.Write([]byte{0x01})
+	hasher.Write(left[:])
+	hasher.Write(right[:])
+	hasher.Sum(hash[:0])
+	return hash
+}

third_party/certspotter/monitor/config.go

@@ -0,0 +1,39 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+)
+
+// IdentifierSink receives the DNS identifiers parsed from every monitored
+// certificate.  This is a CTCLOUD PATCH to upstream certspotter; see
+// CTCLOUD_PATCH.md.  When non-nil it is invoked on the monitoring hot path
+// instead of the watchlist/notification machinery.
+type IdentifierSink interface {
+	HandleIdentifiers(ctx context.Context, identifiers *certspotter.Identifiers)
+}
+
+type Config struct {
+	LogListSource       string
+	State               StateProvider
+	StartAtEnd          bool
+	WatchList           WatchList
+	Verbose             bool
+	HealthCheckInterval time.Duration
+
+	// IdentifierSink is a CTCLOUD PATCH (see CTCLOUD_PATCH.md).  When set,
+	// parsed identifiers are handed directly to it in-process and the normal
+	// watchlist/chain/notification path is skipped.
+	IdentifierSink IdentifierSink
+}

third_party/certspotter/monitor/daemon.go

@@ -0,0 +1,173 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"golang.org/x/sync/errgroup"
+	"log"
+	insecurerand "math/rand"
+	"software.sslmate.com/src/certspotter/loglist"
+	"time"
+)
+
+const (
+	reloadLogListIntervalMin = 30 * time.Minute
+	reloadLogListIntervalMax = 90 * time.Minute
+)
+
+func randomDuration(min, max time.Duration) time.Duration {
+	return min + time.Duration(insecurerand.Int63n(int64(max-min+1)))
+}
+
+func reloadLogListInterval() time.Duration {
+	return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax)
+}
+
+type task struct {
+	log  *loglist.Log
+	stop context.CancelFunc
+}
+
+type daemon struct {
+	config         *Config
+	taskgroup      *errgroup.Group
+	tasks          map[LogID]task
+	logsLoadedAt   time.Time
+	logListToken   *loglist.ModificationToken
+}
+
+func (daemon *daemon) healthCheck(ctx context.Context) error {
+	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
+		errors, err := daemon.config.State.GetErrors(ctx, nil, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
+		var errorsDir string
+		if fsstate, ok := daemon.config.State.(*FilesystemState); ok {
+			errorsDir = fsstate.errorDir(nil)
+		}
+		info := &StaleLogListInfo{
+			Source:        daemon.config.LogListSource,
+			LastSuccess:   daemon.logsLoadedAt,
+			RecentErrors:  errors,
+			ErrorsDir:     errorsDir,
+		}
+		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
+			return fmt.Errorf("error notifying about stale log list: %w", err)
+		}
+	}
+
+	for _, task := range daemon.tasks {
+		if err := healthCheckLog(ctx, daemon.config, task.log); err != nil {
+			return fmt.Errorf("error checking health of log %q: %w", task.log.GetMonitoringURL(), err)
+		}
+	}
+	return nil
+}
+
+func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
+	ctx, cancel := context.WithCancel(ctx)
+	daemon.taskgroup.Go(func() error {
+		defer cancel()
+		err := monitorLogContinously(ctx, daemon.config, ctlog)
+		if daemon.config.Verbose {
+			log.Printf("%s: task stopped with error: %s", ctlog.GetMonitoringURL(), err)
+		}
+		if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
+			return nil
+		} else {
+			return fmt.Errorf("error while monitoring %s: %w", ctlog.GetMonitoringURL(), err)
+		}
+	})
+	return task{log: ctlog, stop: cancel}
+}
+
+func (daemon *daemon) loadLogList(ctx context.Context) error {
+	newLogList, newToken, err := getLogList(ctx, daemon.config.LogListSource, daemon.logListToken)
+	if errors.Is(err, loglist.ErrNotModified) {
+		daemon.logsLoadedAt = time.Now() // so the log list is not reported as stale
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if daemon.config.Verbose {
+		log.Printf("fetched %d logs from %q", len(newLogList), daemon.config.LogListSource)
+	}
+
+	for logID, task := range daemon.tasks {
+		if _, exists := newLogList[logID]; exists {
+			continue
+		}
+		if daemon.config.Verbose {
+			log.Printf("stopping task for log %s", logID.Base64String())
+		}
+		task.stop()
+		delete(daemon.tasks, logID)
+	}
+	for logID, ctlog := range newLogList {
+		if _, isRunning := daemon.tasks[logID]; isRunning {
+			continue
+		}
+		if daemon.config.Verbose {
+			log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.GetMonitoringURL())
+		}
+		daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
+	}
+	daemon.logsLoadedAt = time.Now()
+	daemon.logListToken = newToken
+	return nil
+}
+
+func (daemon *daemon) run(ctx context.Context) error {
+	if err := daemon.config.State.Prepare(ctx); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
+	}
+
+	if err := daemon.loadLogList(ctx); err != nil {
+		return fmt.Errorf("error loading log list: %w", err)
+	}
+
+	reloadLogListTicker := time.NewTicker(reloadLogListInterval())
+	defer reloadLogListTicker.Stop()
+
+	healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval)
+	defer healthCheckTicker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-reloadLogListTicker.C:
+			if err := daemon.loadLogList(ctx); err != nil {
+				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
+			}
+			reloadLogListTicker.Reset(reloadLogListInterval())
+		case <-healthCheckTicker.C:
+			if err := daemon.healthCheck(ctx); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func Run(ctx context.Context, config *Config) error {
+	group, ctx := errgroup.WithContext(ctx)
+	daemon := &daemon{
+		config:    config,
+		taskgroup: group,
+		tasks:     make(map[LogID]task),
+	}
+	group.Go(func() error { return daemon.run(ctx) })
+	return group.Wait()
+}

third_party/certspotter/monitor/discoveredcert.go

@@ -0,0 +1,189 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/pem"
+	"fmt"
+	"strings"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type DiscoveredCert struct {
+	WatchItem    WatchItem
+	LogEntry     *LogEntry
+	Info         *certspotter.CertInfo
+	Chain        []cttypes.ASN1Cert // first entry is the leaf certificate or precertificate
+	ChainError   error              // any error generating or validating Chain; if non-nil, Chain may be partial or incorrect
+	TBSSHA256    [32]byte           // computed over Info.TBS.Raw
+	SHA256       [32]byte           // computed over Chain[0]
+	PubkeySHA256 [32]byte           // computed over Info.TBS.PublicKey.FullBytes
+	Identifiers  *certspotter.Identifiers
+}
+
+type certPaths struct {
+	temporary bool
+
+	certPath string
+	jsonPath string
+	textPath string
+}
+
+func (cert *DiscoveredCert) pemChain() []byte {
+	var buffer bytes.Buffer
+	if cert.ChainError != nil {
+		fmt.Fprintln(&buffer, "Warning: this chain may be incomplete or invalid: ", cert.ChainError)
+	}
+	for _, certBytes := range cert.Chain {
+		if err := pem.Encode(&buffer, &pem.Block{
+			Type:  "CERTIFICATE",
+			Bytes: certBytes,
+		}); err != nil {
+			panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err))
+		}
+	}
+	return buffer.Bytes()
+}
+
+func (cert *DiscoveredCert) json() any {
+	object := map[string]any{
+		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
+		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
+		"dns_names":     cert.Identifiers.DNSNames,
+		"ip_addresses":  cert.Identifiers.IPAddrs,
+	}
+
+	if cert.Info.ValidityParseError == nil {
+		object["not_before"] = cert.Info.Validity.NotBefore
+		object["not_after"] = cert.Info.Validity.NotAfter
+	} else {
+		object["not_before"] = nil
+		object["not_after"] = nil
+	}
+
+	return object
+}
+
+func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
+	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
+		return err
+	}
+	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
+		return err
+	}
+	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
+		return err
+	}
+	return nil
+}
+
+func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
+	env := []string{
+		"EVENT=discovered_cert",
+		"SUMMARY=" + certNotificationSummary(cert),
+		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
+		"LOG_URI=" + cert.LogEntry.Log.GetMonitoringURL(),
+		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
+		"WATCH_ITEM=" + cert.WatchItem.String(),
+		"TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]),
+		"CERT_SHA256=" + hex.EncodeToString(cert.SHA256[:]),
+		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
+		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
+		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
+	}
+
+	if paths != nil {
+		env = append(env, "CERT_FILENAME="+paths.certPath)
+		env = append(env, "JSON_FILENAME="+paths.jsonPath)
+		env = append(env, "TEXT_FILENAME="+paths.textPath)
+	}
+
+	if cert.Info.ValidityParseError == nil {
+		env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String())
+		env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix()))
+		env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339))
+		env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String())
+		env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix()))
+		env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339))
+	} else {
+		env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error())
+	}
+
+	if cert.Info.SubjectParseError == nil {
+		env = append(env, "SUBJECT_DN="+cert.Info.Subject.String())
+	} else {
+		env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error())
+	}
+
+	if cert.Info.IssuerParseError == nil {
+		env = append(env, "ISSUER_DN="+cert.Info.Issuer.String())
+	} else {
+		env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error())
+	}
+
+	if cert.Info.SerialNumberParseError == nil {
+		env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber))
+	} else {
+		env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
+	}
+
+	if cert.ChainError != nil {
+		env = append(env, "CHAIN_ERROR="+cert.ChainError.Error())
+	}
+
+	return env
+}
+
+func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
+	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+
+	fmt.Fprintf(text, "%x:\n", cert.SHA256)
+	for _, dnsName := range cert.Identifiers.DNSNames {
+		writeField("DNS Name", dnsName)
+	}
+	for _, ipaddr := range cert.Identifiers.IPAddrs {
+		writeField("IP Address", ipaddr)
+	}
+	writeField("Pubkey", hex.EncodeToString(cert.PubkeySHA256[:]))
+	if cert.Info.IssuerParseError == nil {
+		writeField("Issuer", cert.Info.Issuer)
+	} else {
+		writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError))
+	}
+	if cert.Info.ValidityParseError == nil {
+		writeField("Not Before", cert.Info.Validity.NotBefore)
+		writeField("Not After", cert.Info.Validity.NotAfter)
+	} else {
+		writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
+		writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
+	}
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.GetMonitoringURL()))
+	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
+	if cert.ChainError != nil {
+		writeField("Error Building Chain", cert.ChainError.Error())
+	}
+	if paths != nil && !paths.temporary {
+		writeField("Filename", paths.certPath)
+	}
+
+	return text.String()
+}
+
+func certNotificationSummary(cert *DiscoveredCert) string {
+	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
+}

third_party/certspotter/monitor/errors.go

@@ -0,0 +1,28 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"log"
+
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
+	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
+		log.Print("unable to notify about error: ", err)
+		if ctlog == nil {
+			log.Print(errToRecord)
+		} else {
+			log.Print(ctlog.GetMonitoringURL(), ": ", errToRecord)
+		}
+	}
+}

third_party/certspotter/monitor/fileutils.go

@@ -0,0 +1,138 @@
+// Copyright (C) 2017, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"slices"
+	"strings"
+)
+
+func randomFileSuffix() string {
+	var randomBytes [12]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:])
+}
+
+func writeSyncFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	if err2 := f.Sync(); err2 != nil && err == nil {
+		err = err2
+	}
+	if err2 := f.Close(); err2 != nil && err == nil {
+		err = err2
+	}
+	return err
+}
+
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	tempname := filename + ".tmp." + randomFileSuffix()
+	if err := writeSyncFile(tempname, data, perm); err != nil {
+		return fmt.Errorf("error writing %s: %w", filename, err)
+	}
+	if err := os.Rename(tempname, filename); err != nil {
+		os.Remove(tempname)
+		return fmt.Errorf("error writing %s: %w", filename, err)
+	}
+	return nil
+}
+
+func writeTextFile(filename string, fileText string, perm os.FileMode) error {
+	return writeFile(filename, []byte(fileText), perm)
+}
+
+func writeJSONFile(filename string, data any, perm os.FileMode) error {
+	fileBytes, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	fileBytes = append(fileBytes, '\n')
+	return writeFile(filename, fileBytes, perm)
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}
+
+// countFiles returns the number of non-hidden files in dirPath whose
+// name ends in suffix.  Returns 0 if dirPath does not exist.
+func countFiles(dirPath string, suffix string) (int, error) {
+	entries, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return 0, nil
+	} else if err != nil {
+		return 0, err
+	}
+	count := 0
+	for _, entry := range entries {
+		if !entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") && strings.HasSuffix(entry.Name(), suffix) {
+			count++
+		}
+	}
+	return count, nil
+}
+
+func tailFile(filename string, linesWanted int) ([]byte, int, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer file.Close()
+	return tail(file, linesWanted, 4096)
+}
+
+func tail(r io.ReadSeeker, linesWanted int, chunkSize int) ([]byte, int, error) {
+	var buf []byte
+	linesGot := 0
+
+	offset, err := r.Seek(0, io.SeekEnd)
+	if err != nil {
+		return nil, 0, err
+	}
+	for offset > 0 {
+		readSize := chunkSize
+		if offset < int64(readSize) {
+			readSize = int(offset)
+		}
+		offset -= int64(readSize)
+		if _, err := r.Seek(offset, io.SeekStart); err != nil {
+			return nil, 0, err
+		}
+		buf = slices.Grow(buf, readSize)
+		copy(buf[readSize:len(buf)+readSize], buf)
+		buf = buf[:len(buf)+readSize]
+		if _, err := io.ReadFull(r, buf[:readSize]); err != nil {
+			return nil, 0, err
+		}
+		for i := readSize; i > 0; i-- {
+			if buf[i-1] == '\n' {
+				if linesGot == linesWanted {
+					return buf[i:], linesGot, nil
+				}
+				linesGot++
+			}
+		}
+	}
+	return buf, linesGot, nil
+}