From e8f7d6bfdf1e8ceb0d369b628a24ea818b394328 Mon Sep 17 00:00:00 2001 From: StasonJatham Date: Tue, 22 Sep 2026 13:13:10 +0200 Subject: [PATCH] ctcloud: CertSpotter-backed CT cloud-resource monitor Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path --- .dockerignore | 6 + .env.example | 16 + .gitignore | 5 + ARCHITECTURE.md | 92 + Dockerfile | 22 + GOAL.md | 1618 +++++++++++++++++ README.md | 253 +++ cmd/ctcloud/main.go | 234 +++ docker-compose.yml | 66 + go.mod | 21 + go.sum | 32 + internal/classify/aws.go | 148 ++ internal/classify/azure.go | 113 ++ internal/classify/classify.go | 198 ++ internal/classify/classify_test.go | 243 +++ internal/classify/cloudflare.go | 34 + internal/classify/digitalocean.go | 39 + internal/classify/gcp.go | 49 + internal/classify/oracle.go | 44 + internal/classify/supabase.go | 45 + internal/config/config.go | 160 ++ internal/config/config_test.go | 53 + internal/dns/cache.go | 99 + internal/dns/pool.go | 119 ++ internal/dns/pool_test.go | 68 + internal/dns/resolver.go | 131 ++ internal/dns/resolver_test.go | 142 ++ internal/metrics/metrics.go | 91 + internal/model/model.go | 219 +++ internal/pipeline/pipeline.go | 345 ++++ internal/pipeline/pipeline_test.go | 236 +++ internal/probe/probe.go | 142 ++ internal/probe/probe_test.go | 45 + internal/store/state.go | 165 ++ internal/store/store.go | 146 ++ internal/store/store_test.go | 66 + migrations/0001_schema.sql | 56 + migrations/embed.go | 21 + .../certspotter/.github/workflows/test.yml | 35 + third_party/certspotter/CHANGELOG.md | 267 +++ third_party/certspotter/CONTRIBUTING | 44 + third_party/certspotter/CTCLOUD_PATCH.md | 53 + third_party/certspotter/LICENSE | 373 ++++ third_party/certspotter/README.md | 183 ++ third_party/certspotter/asn1.go | 78 + third_party/certspotter/asn1time.go | 257 +++ third_party/certspotter/asn1time_test.go | 121 ++ third_party/certspotter/canonicalize.go | 92 + third_party/certspotter/canonicalize_test.go | 49 + .../cmd/certspotter-authorize/.gitignore | 1 + .../cmd/certspotter-authorize/main.go | 202 ++ .../cmd/certspotter-authorize/main_test.go | 190 ++ .../cmd/certspotter-metrics/.gitignore | 1 + .../cmd/certspotter-metrics/main.go | 245 +++ .../certspotter/cmd/certspotter/.gitignore | 1 + .../certspotter/cmd/certspotter/main.go | 290 +++ third_party/certspotter/ctclient/client.go | 204 +++ .../certspotter/ctclient/client_test.go | 77 + third_party/certspotter/ctclient/log.go | 59 + third_party/certspotter/ctclient/rfc6962.go | 272 +++ third_party/certspotter/ctclient/static.go | 401 ++++ .../certspotter/ctclient/static_test.go | 38 + third_party/certspotter/ctcrypto/key.go | 67 + .../certspotter/ctcrypto/signatures.go | 55 + third_party/certspotter/cttypes/certs.go | 121 ++ third_party/certspotter/cttypes/checkpoint.go | 127 ++ .../certspotter/cttypes/checkpoint_test.go | 113 ++ third_party/certspotter/cttypes/helpers.go | 20 + third_party/certspotter/cttypes/logid.go | 69 + third_party/certspotter/cttypes/merkleleaf.go | 213 +++ third_party/certspotter/cttypes/sct.go | 73 + third_party/certspotter/cttypes/signatures.go | 29 + third_party/certspotter/cttypes/sth.go | 37 + third_party/certspotter/cttypes/version.go | 29 + third_party/certspotter/go.mod | 14 + third_party/certspotter/go.sum | 22 + third_party/certspotter/helpers.go | 80 + third_party/certspotter/helpers_test.go | 49 + third_party/certspotter/identifiers.go | 301 +++ third_party/certspotter/loglist/helpers.go | 40 + third_party/certspotter/loglist/load.go | 129 ++ .../certspotter/loglist/mozilla/ctlogs.go | 275 +++ .../loglist/mozilla/ctlogs_test.go | 87 + .../loglist/mozilla/example_test.go | 55 + third_party/certspotter/loglist/schema.go | 158 ++ third_party/certspotter/loglist/validate.go | 57 + third_party/certspotter/man/.gitignore | 1 + third_party/certspotter/man/Makefile | 9 + .../certspotter/man/certspotter-authorize.md | 105 ++ .../certspotter/man/certspotter-metrics.md | 128 ++ .../certspotter/man/certspotter-script.md | 296 +++ third_party/certspotter/man/certspotter.md | 261 +++ .../certspotter/merkletree/collapsed_tree.go | 182 ++ .../certspotter/merkletree/fragment.go | 122 ++ third_party/certspotter/merkletree/hash.go | 77 + third_party/certspotter/monitor/config.go | 39 + third_party/certspotter/monitor/daemon.go | 173 ++ .../certspotter/monitor/discoveredcert.go | 189 ++ third_party/certspotter/monitor/errors.go | 28 + third_party/certspotter/monitor/fileutils.go | 138 ++ third_party/certspotter/monitor/fsstate.go | 416 +++++ .../certspotter/monitor/healthcheck.go | 188 ++ third_party/certspotter/monitor/loglist.go | 45 + third_party/certspotter/monitor/mailutils.go | 34 + third_party/certspotter/monitor/monitor.go | 608 +++++++ third_party/certspotter/monitor/notify.go | 166 ++ third_party/certspotter/monitor/process.go | 151 ++ third_party/certspotter/monitor/state.go | 91 + third_party/certspotter/monitor/statedir.go | 167 ++ third_party/certspotter/monitor/sthdir.go | 178 ++ third_party/certspotter/monitor/watchlist.go | 135 ++ third_party/certspotter/precerts.go | 154 ++ .../certspotter/sequencer/sequencer.go | 151 ++ .../certspotter/sequencer/sequencer_test.go | 195 ++ third_party/certspotter/staticcheck.conf | 1 + .../certspotter/tlstypes/signatures.go | 134 ++ third_party/certspotter/x509.go | 441 +++++ third_party/certspotter/x509_test.go | 135 ++ 118 files changed, 16473 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 ARCHITECTURE.md create mode 100644 Dockerfile create mode 100644 GOAL.md create mode 100644 cmd/ctcloud/main.go create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/classify/aws.go create mode 100644 internal/classify/azure.go create mode 100644 internal/classify/classify.go create mode 100644 internal/classify/classify_test.go create mode 100644 internal/classify/cloudflare.go create mode 100644 internal/classify/digitalocean.go create mode 100644 internal/classify/gcp.go create mode 100644 internal/classify/oracle.go create mode 100644 internal/classify/supabase.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/dns/cache.go create mode 100644 internal/dns/pool.go create mode 100644 internal/dns/pool_test.go create mode 100644 internal/dns/resolver.go create mode 100644 internal/dns/resolver_test.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/model/model.go create mode 100644 internal/pipeline/pipeline.go create mode 100644 internal/pipeline/pipeline_test.go create mode 100644 internal/probe/probe.go create mode 100644 internal/probe/probe_test.go create mode 100644 internal/store/state.go create mode 100644 internal/store/store.go create mode 100644 internal/store/store_test.go create mode 100644 migrations/0001_schema.sql create mode 100644 migrations/embed.go create mode 100644 third_party/certspotter/.github/workflows/test.yml create mode 100644 third_party/certspotter/CHANGELOG.md create mode 100644 third_party/certspotter/CONTRIBUTING create mode 100644 third_party/certspotter/CTCLOUD_PATCH.md create mode 100644 third_party/certspotter/LICENSE create mode 100644 third_party/certspotter/README.md create mode 100644 third_party/certspotter/asn1.go create mode 100644 third_party/certspotter/asn1time.go create mode 100644 third_party/certspotter/asn1time_test.go create mode 100644 third_party/certspotter/canonicalize.go create mode 100644 third_party/certspotter/canonicalize_test.go create mode 100644 third_party/certspotter/cmd/certspotter-authorize/.gitignore create mode 100644 third_party/certspotter/cmd/certspotter-authorize/main.go create mode 100644 third_party/certspotter/cmd/certspotter-authorize/main_test.go create mode 100644 third_party/certspotter/cmd/certspotter-metrics/.gitignore create mode 100644 third_party/certspotter/cmd/certspotter-metrics/main.go create mode 100644 third_party/certspotter/cmd/certspotter/.gitignore create mode 100644 third_party/certspotter/cmd/certspotter/main.go create mode 100644 third_party/certspotter/ctclient/client.go create mode 100644 third_party/certspotter/ctclient/client_test.go create mode 100644 third_party/certspotter/ctclient/log.go create mode 100644 third_party/certspotter/ctclient/rfc6962.go create mode 100644 third_party/certspotter/ctclient/static.go create mode 100644 third_party/certspotter/ctclient/static_test.go create mode 100644 third_party/certspotter/ctcrypto/key.go create mode 100644 third_party/certspotter/ctcrypto/signatures.go create mode 100644 third_party/certspotter/cttypes/certs.go create mode 100644 third_party/certspotter/cttypes/checkpoint.go create mode 100644 third_party/certspotter/cttypes/checkpoint_test.go create mode 100644 third_party/certspotter/cttypes/helpers.go create mode 100644 third_party/certspotter/cttypes/logid.go create mode 100644 third_party/certspotter/cttypes/merkleleaf.go create mode 100644 third_party/certspotter/cttypes/sct.go create mode 100644 third_party/certspotter/cttypes/signatures.go create mode 100644 third_party/certspotter/cttypes/sth.go create mode 100644 third_party/certspotter/cttypes/version.go create mode 100644 third_party/certspotter/go.mod create mode 100644 third_party/certspotter/go.sum create mode 100644 third_party/certspotter/helpers.go create mode 100644 third_party/certspotter/helpers_test.go create mode 100644 third_party/certspotter/identifiers.go create mode 100644 third_party/certspotter/loglist/helpers.go create mode 100644 third_party/certspotter/loglist/load.go create mode 100644 third_party/certspotter/loglist/mozilla/ctlogs.go create mode 100644 third_party/certspotter/loglist/mozilla/ctlogs_test.go create mode 100644 third_party/certspotter/loglist/mozilla/example_test.go create mode 100644 third_party/certspotter/loglist/schema.go create mode 100644 third_party/certspotter/loglist/validate.go create mode 100644 third_party/certspotter/man/.gitignore create mode 100644 third_party/certspotter/man/Makefile create mode 100644 third_party/certspotter/man/certspotter-authorize.md create mode 100644 third_party/certspotter/man/certspotter-metrics.md create mode 100644 third_party/certspotter/man/certspotter-script.md create mode 100644 third_party/certspotter/man/certspotter.md create mode 100644 third_party/certspotter/merkletree/collapsed_tree.go create mode 100644 third_party/certspotter/merkletree/fragment.go create mode 100644 third_party/certspotter/merkletree/hash.go create mode 100644 third_party/certspotter/monitor/config.go create mode 100644 third_party/certspotter/monitor/daemon.go create mode 100644 third_party/certspotter/monitor/discoveredcert.go create mode 100644 third_party/certspotter/monitor/errors.go create mode 100644 third_party/certspotter/monitor/fileutils.go create mode 100644 third_party/certspotter/monitor/fsstate.go create mode 100644 third_party/certspotter/monitor/healthcheck.go create mode 100644 third_party/certspotter/monitor/loglist.go create mode 100644 third_party/certspotter/monitor/mailutils.go create mode 100644 third_party/certspotter/monitor/monitor.go create mode 100644 third_party/certspotter/monitor/notify.go create mode 100644 third_party/certspotter/monitor/process.go create mode 100644 third_party/certspotter/monitor/state.go create mode 100644 third_party/certspotter/monitor/statedir.go create mode 100644 third_party/certspotter/monitor/sthdir.go create mode 100644 third_party/certspotter/monitor/watchlist.go create mode 100644 third_party/certspotter/precerts.go create mode 100644 third_party/certspotter/sequencer/sequencer.go create mode 100644 third_party/certspotter/sequencer/sequencer_test.go create mode 100644 third_party/certspotter/staticcheck.conf create mode 100644 third_party/certspotter/tlstypes/signatures.go create mode 100644 third_party/certspotter/x509.go create mode 100644 third_party/certspotter/x509_test.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d042cfe --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.gitignore +pgdata +*.test +*.out +.gocache diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b92fab1 --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +# Copy to .env and edit. .env is gitignored. + +POSTGRES_USER=ctcloud +POSTGRES_PASSWORD=change-me-strong +POSTGRES_DB=ctcloud + +# Local compose (private bridge network, no TLS): +DATABASE_URL=postgres://ctcloud:change-me-strong@postgres:5432/ctcloud?sslmode=disable + +# Production: use a TLS-enabled database instead, e.g. +# DATABASE_URL=postgres://user:pass@db.example.com:5432/ctcloud?sslmode=verify-full + +DNS_SERVERS=1.1.1.1:53,8.8.8.8:53,1.0.0.1:53 +DNS_WORKERS=16 +LOG_LEVEL=info +ACTIVE_PROBING=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f647318 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/ctcloud +*.test +*.out +pgdata/ +.env diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..ce63b5d --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,92 @@ +# Architecture + +## Data flow + +```text +Certificate Transparency Logs + │ + ▼ + CertSpotter internals fetch / verify / parse (third_party) + (in-process, no stdout/JSON/files) + │ + ▼ + DNS identifiers + │ + ▼ + FAST CLASSIFIER 0 allocs/op, no regexp/JSON/DNS/DB + ┌────┴─────┐ + direct cloud normal + resource hostname + │ │ + │ ▼ + │ DNS/CNAME enrichment bounded workers + TTL cache + │ │ + │ ▼ + │ classify CNAME targets + │ │ + └─────┬──────┘ + ▼ + dedup/enrichment TTL caches; DB uniqueness is final + │ + ▼ + optional safe reachability HTTPS GET / only, OFF by default + ▼ + batch DB writer bounded queue, pgx batches + ▼ + PostgreSQL +``` + +## Stages and what is allowed + +| Stage | May do | Must not do | +| --- | --- | --- | +| CertSpotter fetch/verify/parse | network, crypto, in-memory DER | persist certs, write files | +| Identifier callback | iterate `DNSNames`, call classifier | block, log per certificate, allocate needlessly | +| Classifier | byte/string comparisons, label scan | regexp, JSON, reflection, DNS, HTTP, DB, files, per-id logging | +| Direct queue | bound, non-blocking send | unbounded growth | +| DNS enrichment | DNS via worker pool, TTL cache, CNAME follow | unbounded goroutines, persistence | +| Dedup | memory-bounded TTL caches | unbounded maps | +| Probe (optional) | DNS/TCP/TLS/HTTP HEAD/GET `/` | auth, downloads, mutation, DB or shell access | +| DB writer | batched `pgx.Batch` writes | one synchronous row per event | +| Checkpoints | JSON CT position in `ct_checkpoints`/`ct_sths` | certificate/issuer contents in DB | + +## Concurrency and backpressure + +* CertSpotter runs one monitor goroutine per CT log; it calls the identifier + sink concurrently. +* The sink never blocks: it classifies inline and does a non-blocking send to + either the direct queue or the DNS queue. Full queues increment + `queue_drops_total` / `queue_full_total`. +* Direct matches have a dedicated queue and worker pool, so speculative CNAME + enrichment can be shed first. DNS workers also skip work when the DB queue is + more than half full. +* The DB writer batches by size or `DB_FLUSH_INTERVAL` and flushes on + shutdown. + +## Failure handling + +* DNS and CT network errors are retried by CertSpotter with bounded exponential + backoff; ctcloud counts them (`ct_errors_total`) without logging per event. +* PostgreSQL writes retry a few times with backoff; persistent failure drops + items from the bounded DB queue rather than growing memory. +* The process uses `context` throughout and on `SIGTERM` stops intake, drains + the pipeline, flushes the DB batch and exits within `SHUTDOWN_TIMEOUT`. + +## Metrics and health + +`GET /metrics` exposes counters (`ct_certificates_total`, +`classifier_direct_matches_total`, `classifier_cname_matches_total`, +`dns_*`, `db_*`, `queue_*`) and gauges (`queue_depth{queue="…"}`, +`go_goroutines`, `process_system_bytes`). No hostnames or other high-cardinality +labels are ever used. + +`GET /healthz` is a liveness check. `GET /readyz` verifies the CT monitor is +initialised and PostgreSQL is reachable. + +## Why no files, pipes or subprocesses + +Everything between certificate parsing and classification stays in memory. The +CertSpotter patch hands identifiers directly to Go code, removing serialization +(JSON), IPC (stdout/pipe), process boundaries (subprocess/hooks) and filesystem +I/O (certificate/state files). CT resume state is the only thing persisted, and +it lives in PostgreSQL. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5e13a4b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.25-alpine AS build +WORKDIR /src + +# Cache module downloads (including the local certspotter fork's deps). +COPY go.mod go.sum ./ +COPY third_party/certspotter/go.mod third_party/certspotter/go.sum ./third_party/certspotter/ +RUN go mod download + +COPY . . + +# Static binary: no cgo, no shell, no writable filesystem required. +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath -ldflags="-s -w" \ + -o /out/ctcloud ./cmd/ctcloud + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/ctcloud /ctcloud +USER nonroot:nonroot +EXPOSE 9090 +ENTRYPOINT ["/ctcloud"] diff --git a/GOAL.md b/GOAL.md new file mode 100644 index 0000000..d6731ea --- /dev/null +++ b/GOAL.md @@ -0,0 +1,1618 @@ +# Goal + +Build a production-ready, extremely resource-efficient Certificate Transparency monitoring service in **Go**, running entirely in **Docker**, using **CertSpotter's existing CT ingestion/parsing implementation**. + +The purpose is to continuously consume CT logs, identify cloud storage/database/data-service resources, enrich candidates via DNS/CNAME resolution, optionally perform minimal safe reachability checks, and persist only useful enriched findings into PostgreSQL. + +The primary design objective is: + +> **Maximum throughput and minimum CPU, memory, allocation, disk, serialization, and IPC overhead.** + +Do not build another generic CertStream consumer. Reuse CertSpotter's CT downloading, validation, parsing, and log-monitoring logic. + +--- + +# Critical architectural requirement + +DO NOT use CertSpotter like this: + +```text +certspotter + → stdout + → JSON + → pipe + → parser +``` + +DO NOT use: + +```text +CertSpotter + → JSON_FILENAME + → filesystem + → hook + → subprocess + → parser +``` + +Instead modify/integrate CertSpotter so parsed certificate DNS identifiers are handed directly to our Go code **inside the same process**. + +Target architecture: + +```text +Certificate Transparency Logs + │ + ▼ + CertSpotter internals + fetch / verify / parse + │ + ▼ + DNS identifiers + │ + ▼ + FAST CLASSIFIER + │ + ┌─────┴─────┐ + │ │ + direct cloud normal + resource hostname + │ │ + │ ▼ + │ DNS/CNAME enrichment + │ │ + │ ▼ + │ classify CNAME targets + │ │ + └─────┬─────┘ + ▼ + dedup/enrichment + │ + ▼ + optional safe reachability + │ + ▼ + batch DB writer + │ + ▼ + PostgreSQL +``` + +Everything between certificate parsing and candidate classification must remain in memory. + +--- + +# CertSpotter integration + +Use the upstream CertSpotter Go code as the CT engine. + +DO NOT reimplement CT log handling unless absolutely required. + +Inspect the current CertSpotter source before modifying anything. + +Create the **smallest possible maintainable patch** that exposes parsed certificate identifiers to our code. + +Preferred abstraction: + +```go +type IdentifierSink interface { + HandleIdentifier(ctx context.Context, identifier Identifier) +} +``` + +or an equally efficient callback. + +The goal is: + +```text +CertSpotter parsed certificate + ↓ +direct Go function call + ↓ +classifier +``` + +There must be: + +```text +NO stdout handoff +NO JSON handoff +NO hook scripts +NO subprocesses +NO certificate files +NO JSON files +NO temporary certificate files +``` + +Certificate DER/metadata may exist temporarily in memory while CertSpotter processes it, but must never be persisted. + +Do not use CertSpotter's normal certificate/output persistence functionality. + +Preserve upstream licensing/notices and keep our CertSpotter changes isolated so upstream changes can later be rebased. + +--- + +# Runtime filesystem policy + +The application must not persist certificates, SAN lists, JSON events, queues, enrichment payloads, or findings to the filesystem. + +No filesystem-based spool. + +No file-based queue. + +No external configuration files. + +No pattern YAML at runtime. + +No file uploads or file ingestion API. + +Configuration comes from environment variables. + +Provider patterns should be compiled into the binary. + +The Docker application container should require **no persistent volume**. + +Run with a read-only root filesystem. + +--- + +# CT checkpoint persistence + +We still need restart-safe CT monitoring. + +Do NOT persist CertSpotter checkpoint/state files locally. + +Store only the minimum required CT progress/checkpoint metadata in PostgreSQL. + +Example conceptual table: + +```text +ct_checkpoints + +log_id +tree_size +position / index +updated_at +``` + +Inspect CertSpotter's real state model and preserve whatever fields are actually required. + +If CertSpotter currently assumes filesystem-backed state, introduce a minimal storage abstraction: + +```go +type StateStore interface { + Load(...) + Save(...) +} +``` + +Implement: + +```text +PostgresStateStore +``` + +Do not store certificate contents in it. + +--- + +# Hot path requirements + +The certificate identifier classifier is the most performance-sensitive part. + +It must have: + +```text +NO regexp +NO JSON +NO reflection +NO interface{} maps +NO database access +NO DNS +NO HTTP +NO filesystem +NO subprocess +NO logging per identifier +NO unnecessary heap allocations +NO string splitting into []string +NO fmt.Sprintf +``` + +Target: + +```text +CertSpotter DNS identifier + ↓ +normalize + ↓ +cheap provider suffix rejection + ↓ +manual label/suffix classifier + ↓ +small struct + ↓ +bounded channel/ring +``` + +Use: + +```go +strings.HasSuffix +strings.HasPrefix +strings.Index / LastIndexByte +direct byte/string comparisons +manual backwards DNS-label scanning +small uint8 enums +``` + +Do not use regex. + +Do not use Hyperscan. + +Do not use Aho-Corasick unless benchmarking proves it faster for this actual tiny pattern set. + +For this number of signatures, direct suffix/label matching should be faster and simpler. + +--- + +# Allocation requirements + +The classifier itself should ideally achieve: + +```text +0 allocs/op +``` + +after receiving the hostname string. + +Avoid lowercasing every hostname if unnecessary. + +Fast-path ASCII lowercase names directly. + +Only normalize when uppercase characters are actually present. + +Avoid: + +```go +strings.Split() +``` + +because it allocates. + +Parse DNS labels by indexes. + +--- + +# Core data structures + +Use compact enums. + +Example: + +```go +type Provider uint8 + +const ( + ProviderUnknown Provider = iota + ProviderAWS + ProviderAzure + ProviderGCP + ProviderCloudflare + ProviderDigitalOcean + ProviderSupabase + ProviderOracle +) + +type Service uint8 + +const ( + ServiceUnknown Service = iota + + ServiceObjectStorage + ServiceS3 + ServiceBlobStorage + ServiceDatabase + ServicePostgres + ServiceMySQL + ServiceRDS + ServiceDocumentDB + ServiceRedshift + ServiceOpenSearch + ServiceNeptune + ServiceRedis + ServiceMemoryDB + ServiceCosmosDB + ServiceFirebaseDB +) +``` + +A hot-path match should look roughly like: + +```go +type Candidate struct { + Hostname string + Provider Provider + Service Service + Source Source +} +``` + +Keep hot-path structs small. + +Do not attach large certificate structures to downstream jobs. + +--- + +# Source tracking + +We need to distinguish: + +```text +DIRECT_CT +CNAME +DNS_ALIAS +``` + +Example: + +```text +assets.company.com + CNAME +company-assets.s3.eu-central-1.amazonaws.com +``` + +should produce an enriched record containing both: + +```text +original_hostname = assets.company.com +provider_hostname = company-assets.s3.eu-central-1.amazonaws.com +source = CNAME +provider = AWS +service = S3 +``` + +--- + +# Provider classification + +Implement dedicated provider classifiers. + +## AWS + +Recognize resource-specific forms including: + +```text +# S3 +.s3.amazonaws.com +.s3..amazonaws.com +.s3.dualstack..amazonaws.com +.s3-website..amazonaws.com +.s3-website-.amazonaws.com + +# Access Points +-.s3-accesspoint..amazonaws.com +*.s3-accesspoint.dualstack..amazonaws.com +*.s3-accesspoint-fips..amazonaws.com + +# Multi-region +*.mrap.accesspoint.s3-global.amazonaws.com + +# S3 Express +*.s3express-..amazonaws.com + +# RDS / Aurora / RDS Proxy +*.rds.amazonaws.com + +# DocumentDB +*.docdb.amazonaws.com + +# Redshift +*.redshift.amazonaws.com +*.redshift-serverless.amazonaws.com + +# OpenSearch +search-*.es.amazonaws.com +vpc-*.es.amazonaws.com + +# Neptune +*.neptune.amazonaws.com + +# MemoryDB +*.memorydb..amazonaws.com + +# ElastiCache +*.cache.amazonaws.com +``` + +Further classify AWS subtypes where cheap. + +Examples: + +```text +cluster-* → Aurora/cluster +cluster-ro-* → reader +proxy-* → RDS proxy + +search-* → OpenSearch public endpoint form +vpc-* → OpenSearch VPC endpoint form +``` + +Do not treat generic AWS service API endpoints as customer resources. + +Examples to reject: + +```text +s3.eu-central-1.amazonaws.com +dynamodb.eu-central-1.amazonaws.com +redshift.eu-central-1.amazonaws.com +``` + +--- + +# Azure + +Recognize: + +```text +*.blob.core.windows.net +*.dfs.core.windows.net +*.file.core.windows.net +*.queue.core.windows.net +*.table.core.windows.net +*.web.core.windows.net + +*.database.windows.net +*.postgres.database.azure.com +*.mysql.database.azure.com + +*.documents.azure.com +*.mongo.cosmos.azure.com +*.cassandra.cosmos.azure.com +*.gremlin.cosmos.azure.com + +*.redis.cache.windows.net +*.redis.azure.net + +*.search.windows.net +``` + +Require a customer/resource-specific prefix. + +Do not classify bare provider service endpoints as resource findings. + +--- + +# Google Cloud / Firebase + +Recognize: + +```text +.storage.googleapis.com + +.firebaseio.com + +..firebasedatabase.app + +*.appspot.com +``` + +Do not classify: + +```text +storage.googleapis.com +``` + +itself as a resource. + +--- + +# Cloudflare + +Recognize: + +```text +.r2.cloudflarestorage.com +*.r2.dev +``` + +Classify: + +```text +r2.cloudflarestorage.com → Cloudflare R2 API/resource endpoint +r2.dev → public R2 development/public endpoint +``` + +Remember that arbitrary custom R2 domains will only be discovered through DNS/CNAME enrichment. + +--- + +# DigitalOcean + +Recognize: + +```text +..digitaloceanspaces.com +..cdn.digitaloceanspaces.com + +*.db.ondigitalocean.com +``` + +Do not treat: + +```text +.digitaloceanspaces.com +``` + +as a customer bucket. + +--- + +# Supabase + +Recognize: + +```text +.supabase.co +db..supabase.co +aws--.pooler.supabase.com +``` + +Classify direct PostgreSQL separately where possible: + +```text +db..supabase.co +``` + +--- + +# Oracle Cloud / OCI + +Recognize: + +```text +.objectstorage..oci.customer-oci.com +.swiftobjectstorage..oci.customer-oci.com +``` + +Also recognize generic OCI storage service endpoints but mark them: + +```text +resource_specific = false +``` + +Examples: + +```text +objectstorage..oraclecloud.com +swiftobjectstorage..oraclecloud.com +``` + +These should normally not become high-value findings because the bucket name exists in the path rather than the hostname. + +--- + +# Classification quality + +Every matcher must determine: + +```go +type Match struct { + Provider Provider + Service Service + ResourceSpecific bool + PublicSemantic PublicSemantic +} +``` + +`PublicSemantic` should distinguish cases where hostname format itself carries useful public/private information. + +Example: + +```text +AWS search-*.es.amazonaws.com + → public network endpoint form + +AWS vpc-*.es.amazonaws.com + → VPC endpoint form + +Cloudflare *.r2.dev + → explicitly public URL form +``` + +Do NOT equate: + +```text +resource exists +``` + +with: + +```text +data is publicly accessible +``` + +--- + +# Direct-match path + +Every SAN/DNS identifier received from CertSpotter should first be passed through the provider classifier. + +Direct provider matches should be extremely cheap: + +```text +SAN + ↓ +provider suffix rejection + ↓ +specific classifier + ↓ +Candidate +``` + +Direct matches go immediately into enrichment. + +--- + +# CNAME discovery + +Provider-native names alone are insufficient because certificates frequently contain custom hostnames: + +```text +files.example.com + ↓ CNAME +foo.blob.core.windows.net +``` + +or: + +```text +db.example.com + ↓ CNAME +prod.abc.eu-central-1.rds.amazonaws.com +``` + +Therefore also support DNS enrichment of CT hostnames. + +Pipeline: + +```text +CT hostname + ↓ +dedupe/cache + ↓ +DNS + ↓ +follow CNAME chain + ↓ +run classifier against EVERY CNAME target + ↓ +candidate +``` + +Follow CNAME chains with: + +```text +maximum depth: configurable +default: 8 +``` + +Detect loops. + +--- + +# DNS resolver + +DNS enrichment must be optimized heavily. + +Use a fixed worker pool. + +Do not spawn unbounded goroutines. + +Use either Go's resolver or `miekg/dns`, whichever benchmarks better for our workload. + +Requirements: + +```text +connection reuse where applicable +bounded concurrency +context cancellation +timeouts +positive TTL cache +negative TTL cache +CNAME cache +duplicate suppression +``` + +The DNS cache is memory-only. + +No persistence. + +Use a memory-bounded implementation. + +It must have a configurable maximum number of entries. + +If implementing a custom sharded TTL cache is simpler/faster than bringing in a heavy dependency, prefer the custom implementation. + +--- + +# Backpressure + +Do not allow enrichment to stall CertSpotter indefinitely. + +Use bounded queues. + +Separate: + +```text +direct-match queue + +DNS-enrichment queue + +DB-write queue +``` + +Expose counters for: + +```text +queue depth +queue full events +dropped optional DNS enrichments +direct matches +CNAME matches +DNS errors +DB errors +``` + +Direct provider matches should receive higher priority than speculative CNAME enrichment. + +If load shedding becomes necessary, drop optional CNAME enrichment before dropping direct cloud-resource matches. + +--- + +# Deduplication + +Certificates and CT logs produce duplicates. + +Perform cheap in-memory dedup before enrichment. + +Key approximately: + +```text +normalized hostname +``` + +After classification: + +```text +provider + service + provider_hostname +``` + +Use memory-bounded TTL caches. + +Do not keep an unbounded map. + +Database uniqueness is the final deduplication layer. + +--- + +# Enrichment model + +For useful candidates collect, when cheap: + +```text +original CT hostname +provider hostname +provider +service +region if extractable +resource identifier if extractable + +first seen +last seen + +CNAME chain + +A records +AAAA records + +public DNS resolution boolean + +source: + direct_ct + cname + +resource_specific + +public_semantic +``` + +Do not store the full certificate unless there is a concrete future requirement. + +Optional useful certificate metadata: + +```text +certificate fingerprint +not_before +not_after +issuer CN +CT log ID +``` + +Only include these if CertSpotter already has them available without additional expensive parsing/copying. + +--- + +# Safe reachability checks + +Make active network checks optional: + +```text +ACTIVE_PROBING=false +``` + +by default. + +When enabled, only perform minimal non-destructive checks on infrastructure the operator is authorized to assess. + +Allowed checks: + +```text +DNS resolution +TCP connect +TLS handshake +HTTP HEAD +HTTP GET / +``` + +No authentication attempts. + +No password guessing. + +No object downloading. + +No mutation. + +No PUT/POST/DELETE. + +No database queries. + +No exploitation. + +No brute force. + +For HTTP-based storage endpoints, use response status, headers, and provider error codes to classify: + +```text +reachable +not_found +auth_required +public_read +unknown +``` + +Do not claim something is "exposed" merely because its hostname exists. + +--- + +# Exposure state + +Use an explicit enum: + +```go +type ExposureState uint8 + +const ( + ExposureUnknown ExposureState = iota + ExposurePublicDNS + ExposureReachable + ExposureAuthRequired + ExposurePublicRead + ExposurePrivate + ExposureNotFound +) +``` + +Keep factual observations separate from conclusions. + +--- + +# PostgreSQL + +Use PostgreSQL with: + +```text +pgx/v5 +pgxpool +``` + +Do not write one row per network event synchronously. + +Use an asynchronous bounded DB writer with batching. + +Use `COPY FROM` or efficient pgx batches where appropriate. + +Suggested schema: + +```text +cloud_assets +------------ +id +provider +service +hostname +provider_hostname +resource_identifier +region +resource_specific +public_semantic +first_seen +last_seen +exposure_state + +UNIQUE(provider, service, provider_hostname) +``` + +and: + +```text +observations +------------ +asset_id +observed_at +source_hostname +source_type +cname_chain +ipv4 +ipv6 +cert_fingerprint +ct_log_id +``` + +Exact schema may be improved if benchmarking/query requirements justify it. + +Do not store observations that provide no useful information. + +Avoid unnecessary JSONB if typed PostgreSQL arrays/columns are more efficient. + +--- + +# Checkpoints + +Separate CT processing state from cloud-asset data. + +Example: + +```text +ct_checkpoints +-------------- +log_id PRIMARY KEY +state +updated_at +``` + +Persist only enough to resume monitoring reliably. + +Batch/debounce checkpoint writes rather than writing on every certificate if CertSpotter semantics permit. + +Correctness must be preserved. + +--- + +# Logging + +Logging must never be on the certificate hot path. + +Do not log every certificate. + +Do not log every SAN. + +Do not log every rejected hostname. + +Log: + +```text +startup +shutdown +fatal errors +persistent DB errors +periodic aggregated statistics +``` + +Default periodic stats interval: + +```text +60 seconds +``` + +Use structured logging but keep it lightweight. + +--- + +# Metrics + +Expose lightweight Prometheus-compatible metrics on an optional HTTP listener. + +At minimum: + +```text +ct_certificates_total +ct_identifiers_total + +classifier_direct_matches_total +classifier_cname_matches_total + +dns_queries_total +dns_cache_hits_total +dns_errors_total + +candidates_total + +db_rows_total +db_errors_total + +queue_depth +queue_drops_total + +process memory +goroutines +``` + +Do not add expensive high-cardinality labels. + +Never label metrics by hostname. + +--- + +# Health + +Provide: + +```text +GET /healthz +GET /readyz +``` + +`readyz` should verify: + +```text +CT monitor initialized +database reachable +critical worker pipelines running +``` + +--- + +# Configuration + +Environment variables only. + +Suggested configuration: + +```text +DATABASE_URL= + +DNS_WORKERS= +DNS_TIMEOUT= +DNS_CACHE_SIZE= +DNS_MAX_CNAME_DEPTH= + +DIRECT_QUEUE_SIZE= +DNS_QUEUE_SIZE= +DB_QUEUE_SIZE= + +DB_BATCH_SIZE= +DB_FLUSH_INTERVAL= + +ACTIVE_PROBING=false +PROBE_WORKERS= +PROBE_TIMEOUT= + +METRICS_ADDR=:9090 +LOG_LEVEL=info +STATS_INTERVAL=60s +``` + +Provide sensible defaults. + +Validate configuration on startup. + +--- + +# Docker + +Create a production Docker setup. + +Use a multi-stage Dockerfile. + +Build a static/minimal Go binary where possible. + +Final image should be minimal, for example distroless or equivalent. + +Run as non-root. + +Requirements: + +```text +read-only root filesystem +no application volume +no CertSpotter volume +no certificate storage +no writable application directory +``` + +Use: + +```text +tmpfs +``` + +only if absolutely unavoidable; preferably the application should not require it. + +Provide: + +```text +Dockerfile +docker-compose.yml +``` + +Docker Compose should contain: + +```text +ctcloud +postgres +``` + +The application itself must remain stateless apart from PostgreSQL. + +--- + +# Code organization + +Keep the project simple. + +Suggested layout: + +```text +cmd/ctcloud/ + main.go + +internal/ + certspotter/ + classify/ + classify.go + aws.go + azure.go + gcp.go + cloudflare.go + digitalocean.go + supabase.go + oracle.go + + dns/ + probe/ + pipeline/ + store/ + metrics/ + config/ + model/ + +migrations/ +Dockerfile +docker-compose.yml +README.md +ARCHITECTURE.md +``` + +If CertSpotter is vendored/forked, keep upstream changes clearly separated. + +Do not introduce microservices. + +This should be **one Go application process** plus PostgreSQL. + +No Kafka. + +No Redis. + +No RabbitMQ. + +No external queue. + +No Elasticsearch. + +No unnecessary infrastructure. + +--- + +# Performance strategy + +Optimize in this order: + +1. eliminate I/O from hot path +2. eliminate serialization +3. eliminate process boundaries +4. eliminate allocations +5. cheap provider rejection +6. cheap suffix/label matching +7. bounded worker pools +8. DNS caching +9. deduplication before enrichment +10. batched DB writes + +Do not prematurely implement complex SIMD/native libraries unless benchmarks demonstrate an actual bottleneck. + +--- + +# Benchmarks + +Add benchmarks for classification. + +Example: + +```go +func BenchmarkClassifyAWS(b *testing.B) +func BenchmarkClassifyAzure(b *testing.B) +func BenchmarkClassifyMixed(b *testing.B) +func BenchmarkRejectNormalDomain(b *testing.B) +``` + +Benchmark realistic input: + +```text +www.example.com +foo.bar.example.org +bucket.s3.eu-central-1.amazonaws.com +foo.blob.core.windows.net +db.example.supabase.co +project.firebaseio.com +search-example-abc.eu-west-1.es.amazonaws.com +``` + +Important metric: + +```text +allocs/op +``` + +Classifier target: + +```text +0 allocs/op +``` + +Do not make CI depend on an absolute nanosecond threshold because runner performance varies. + +Report benchmark numbers in README. + +--- + +# Tests + +Implement table-driven tests covering every provider format. + +Test: + +```text +positive resource matches +generic provider endpoints rejected +uppercase handling +trailing dot +wildcards +invalid hostnames +very long hostnames +CNAME chains +CNAME loops +duplicate events +IPv4/IPv6 +context cancellation +database retries +queue saturation +``` + +Particularly ensure these DO NOT produce resource findings: + +```text +storage.googleapis.com +s3.eu-central-1.amazonaws.com +redshift.eu-central-1.amazonaws.com +fra1.digitaloceanspaces.com +``` + +while: + +```text +bucket.storage.googleapis.com +bucket.s3.eu-central-1.amazonaws.com +cluster.id.eu-central-1.redshift.amazonaws.com +bucket.fra1.digitaloceanspaces.com +``` + +do. + +--- + +# Wildcards + +Handle certificate wildcards carefully. + +Example: + +```text +*.example.com +``` + +must not become a literal DNS query. + +Do not interpret provider-wide wildcard certificates such as: + +```text +*.blob.core.windows.net +``` + +as discovered customer resources. + +Track wildcard certificates separately or ignore them for resource-specific classification unless a concrete hostname can be derived from another SAN. + +--- + +# Graceful shutdown + +Use contexts throughout. + +On SIGTERM: + +```text +stop accepting new enrichment +finish bounded in-flight enrichment +flush DB batch +save CT checkpoint +close DB +exit +``` + +Shutdown must have a maximum timeout. + +Docker should be able to stop the container cleanly. + +--- + +# Failure handling + +The program must survive transient: + +```text +DNS failures +network timeouts +CT connection failures +PostgreSQL connection resets +``` + +with bounded exponential backoff and jitter. + +Never retry in a tight loop. + +If PostgreSQL remains unavailable and queues approach capacity, prefer controlled backpressure/load shedding instead of unbounded memory growth. + +--- + +# Security + +Run as non-root. + +No shell needed in final container. + +No exposed debug endpoints. + +Set HTTP server timeouts. + +Validate all hostname lengths. + +Treat all CT data as untrusted input. + +Never execute certificate-controlled values. + +Never construct shell commands. + +Use parameterized SQL only. + +--- + +# README + +Document: + +```text +what the application does +architecture +why CertSpotter is embedded/patched +why stdout/hooks/files are deliberately avoided +supported provider patterns +configuration +Docker startup +database schema +performance characteristics +benchmark results +limitations +``` + +Clearly explain: + +```text +A detected provider endpoint does NOT automatically mean +the underlying data is publicly accessible. +``` + +--- + +# ARCHITECTURE.md + +Include a concise data-flow diagram: + +```text +CT + ↓ +CertSpotter + ↓ +Identifier callback + ↓ +Direct classifier ───────────────┐ + ↓ non-match │ +DNS/CNAME │ + ↓ │ +Provider classifier │ + └───────────────┬───────────────┘ + ↓ + dedupe + ↓ + enrichment + ↓ + optional probing + ↓ + DB batch + ↓ + PostgreSQL +``` + +Explain which operations are allowed in each stage. + +--- + +# Implementation priorities + +Implement in this order: + +## Phase 1 + +```text +CertSpotter integration +in-memory identifier callback +provider classifier +classification tests +benchmarks +``` + +Do not continue until the classifier has zero unnecessary allocations. + +## Phase 2 + +```text +bounded pipeline +dedup +DNS resolver +CNAME following +DNS cache +``` + +## Phase 3 + +```text +PostgreSQL schema +batch writer +Postgres CT checkpoint state +``` + +## Phase 4 + +```text +optional safe probing +metrics +health endpoints +``` + +## Phase 5 + +```text +Docker hardening +benchmarks +documentation +integration tests +``` + +--- + +# Important implementation rule + +Do not blindly follow this design if inspection of the current CertSpotter source reveals a more direct or efficient integration point. + +The goal is not to satisfy a predetermined class hierarchy. + +The goal is: + +```text +reuse CertSpotter ++ +remove output/storage overhead ++ +receive parsed DNS identifiers directly ++ +classify with essentially negligible CPU cost ++ +only enrich potentially useful data ++ +persist final useful results efficiently +``` + +If CertSpotter already exposes an internal abstraction that accomplishes this cleanly, use it rather than creating another layer. + +--- + +# Things you must NOT do + +Do not: + +```text +write certificates to disk +write JSON events to disk +read JSON event files +call CertSpotter through exec.Command +parse CertSpotter stdout +use hooks +fork a process per certificate +use regex in the classifier +use shell pipelines +store every CT certificate +store every SAN +create unbounded goroutines +create unbounded queues +create unbounded maps +write each result individually to PostgreSQL +introduce Kafka/Redis just for queues +build multiple microservices +``` + +--- + +# Definition of done + +The project is complete when: + +1. A single Dockerized Go process continuously consumes CT logs using CertSpotter internals. +2. CertSpotter never writes certificate/result files. +3. Parsed DNS identifiers are transferred directly into our classifier via Go calls. +4. The classifier covers AWS, Azure, GCP/Firebase, Cloudflare R2, DigitalOcean, Supabase and OCI patterns above. +5. Direct provider resources are detected immediately. +6. CNAME chains are asynchronously enriched and classified. +7. Duplicate processing is bounded and minimized. +8. Only useful resource findings reach PostgreSQL. +9. CT resume/checkpoint state lives in PostgreSQL rather than local files. +10. Classifier benchmarks demonstrate zero unnecessary allocations. +11. Resource usage remains bounded under overload. +12. Docker requires no application persistent volume. +13. `go test ./...` passes. +14. `go test -race ./...` passes. +15. `go vet ./...` passes. +16. Docker build succeeds. +17. The README explains exactly how to run and benchmark it. + +Before declaring completion, inspect the resulting program under load and remove any unnecessary: + +```text +allocations +copies +serialization +logging +locking +filesystem access +goroutine creation +database round trips +``` + +Favor simple, measurable performance over architectural complexity. diff --git a/README.md b/README.md index e69de29..ac2e014 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,253 @@ +# ctcloud + +`ctcloud` continuously consumes Certificate Transparency (CT) logs, finds +cloud storage/database/data-service resources referenced by certificates, +enriches candidates through DNS/CNAME resolution, and persists only useful +findings to PostgreSQL. + +It is a **single stateless Go process plus PostgreSQL**. CertSpotter's real CT +fetching, validation and parsing code is embedded and called **in-process**; +there is no stdout handoff, no JSON handoff, no hooks, no subprocesses and no +certificate files. + +> A detected provider endpoint **does not** mean the underlying data is publicly +> accessible. `ctcloud` reports facts (hostname existed in CT, resolved, answered +> a request) separately from conclusions (`exposure_state`). Most findings are +> simply an endpoint that exists. + +## How it works + +```text +CT logs ──► CertSpotter internals (fetch/verify/parse) + │ direct Go call: IdentifierSink.HandleIdentifiers + ▼ + fast classifier (0 allocs/op, no regexp/JSON/DNS) + ┌──────┴───────┐ + direct match non-match + │ ▼ + │ DNS/CNAME enrichment + │ (followed + classified) + └──────┬───────┘ + ▼ + dedup + bounded queues + ▼ + optional safe reachability + ▼ + batched PostgreSQL writer + ▼ + PostgreSQL +``` + +See [ARCHITECTURE.md](ARCHITECTURE.md) for per-stage rules. + +## Why CertSpotter is embedded (and patched) + +Re-implementing CT log handling is a large, security-sensitive job. Upstream +CertSpotter already does log-list handling, STH fetching/verification, Merkle +tree maintenance, entry parsing and certificate reconstruction. + +The only change is a two-hunk patch in `third_party/certspotter` that adds an +`IdentifierSink` callback. After a certificate is parsed, the identifiers are +handed to ctcloud with a direct Go function call and the watchlist/chain/ +notification machinery is skipped: + +```go +if config.IdentifierSink != nil { + config.IdentifierSink.HandleIdentifiers(ctx, identifiers) + return nil +} +``` + +See [`third_party/certspotter/CTCLOUD_PATCH.md`](third_party/certspotter/CTCLOUD_PATCH.md). +This deliberately avoids stdout piping, JSON files, hook scripts, subprocesses +and temporary certificate files, which are all slow, serializing and +filesystem-bound. Upstream licensing/notices are preserved and the patch is +isolated so it can be rebased. + +## Supported provider patterns + +**AWS** — S3 (`.s3[.dualstack]..amazonaws.com`, +`s3-website[-]`), S3 access points (`s3-accesspoint[-fips]`, +`dualstack`), multi-region access points (`mrap.accesspoint.s3-global`), S3 +Express (`s3express-*`), RDS/Aurora/proxy (`*.rds.amazonaws.com`), DocumentDB, +Redshift (provisioned + serverless), OpenSearch (`search-*` public / `vpc-*` +VPC), Neptune, MemoryDB, ElastiCache. Generic API endpoints +(`s3.eu-central-1.amazonaws.com`, `dynamodb.*`, `redshift.`, +`rds.`, …) are rejected. + +**Azure** — `blob|dfs|file|queue|table|web.core.windows.net`, +`*.database.windows.net`, `*.postgres|mysql.database.azure.com`, +`*.documents.azure.com`, `*.{mongo,cassandra,gremlin}.cosmos.azure.com`, +`*.redis.cache.windows.net`, `*.redis.azure.net`, `*.search.windows.net`. +Bare service endpoints are rejected. + +**Google Cloud / Firebase** — `.storage.googleapis.com`, +`.firebaseio.com`, `..firebasedatabase.app`, +`.appspot.com`. `storage.googleapis.com` itself is rejected. + +**Cloudflare R2** — `.r2.cloudflarestorage.com`; `.r2.dev` +is flagged as an explicitly public URL form. + +**DigitalOcean** — `.[.cdn].digitaloceanspaces.com`, +`*.db.ondigitalocean.com`. `.digitaloceanspaces.com` is rejected. + +**Supabase** — `.supabase.co`, `db..supabase.co` (PostgreSQL), +`aws--.pooler.supabase.com`. + +**Oracle Cloud (OCI)** — `.objectstorage|swiftobjectstorage..oci.customer-oci.com` +(resource-specific), and the generic `objectstorage..oraclecloud.com` +forms (`resource_specific = false`; not stored as findings). + +Provider-specific details live in `internal/classify/`. + +## Configuration + +Environment variables only; no config files, no runtime pattern files. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `DATABASE_URL` | *(required)* | PostgreSQL connection string | +| `DB_MAX_CONNS` | `8` | pgx pool size | +| `DNS_SERVERS` | `1.1.1.1:53,8.8.8.8:53,1.0.0.1:53` | comma-separated resolvers (round-robin + failover); `system` for the host resolver | +| `DNS_WORKERS` | `16` | DNS enrichment workers | +| `DNS_TIMEOUT` | `5s` | per-lookup timeout | +| `DNS_CACHE_SIZE` | `1048576` | max DNS cache entries | +| `DNS_MAX_CNAME_DEPTH` | `8` | max CNAME hops | +| `DNS_POSITIVE_TTL` / `DNS_NEGATIVE_TTL` | `10m` / `1m` | DNS cache TTLs | +| `DIRECT_QUEUE_SIZE` | `8192` | direct-match queue capacity | +| `DNS_QUEUE_SIZE` | `65536` | DNS-enrichment queue capacity | +| `DB_QUEUE_SIZE` | `8192` | DB-write queue capacity | +| `DB_BATCH_SIZE` | `500` | findings per write batch | +| `DB_FLUSH_INTERVAL` | `2s` | max time before a batch is flushed | +| `ACTIVE_PROBING` | `false` | enable optional reachability checks | +| `PROBE_WORKERS` / `PROBE_TIMEOUT` | `4` / `5s` | probe pool | +| `DEDUP_TTL` | `1h` | hostname/finding dedup window | +| `METRICS_ADDR` | `:9090` | metrics/health listener | +| `LOG_LEVEL` | `info` | `debug\|info\|warn\|error` | +| `STATS_INTERVAL` | `60s` | aggregated stats log interval | +| `LOG_LIST_SOURCE` | Google log list v3 | certspotter log list URL | +| `START_AT_END` | `false` | skip existing backlog on first run | +| `SHUTDOWN_TIMEOUT` | `20s` | graceful shutdown budget | + +## Running with Docker + +```sh +cp .env.example .env # set a strong POSTGRES_PASSWORD and matching DATABASE_URL +docker compose up --build +``` + +That starts `postgres` and the stateless `ctcloud` container. The application +container runs as non-root, has a read-only root filesystem, drops all Linux +capabilities, sets CPU/memory/pids limits and mounts no application volume. The +only persistent state is PostgreSQL (CT checkpoints + cloud assets). + +Endpoints: `GET /metrics`, `GET /healthz`, `GET /readyz` on port `9090`, bound +to `127.0.0.1` only. The listener has no authentication, so scrape it locally or +from a sidecar and never expose it publicly. + +## Production notes + +* **Secrets** — database credentials live in `.env` (gitignored); `.env.example` + is the template. Inject the same variables from your orchestrator's secret + store in production. +* **Database TLS** — the local compose stack uses `sslmode=disable` because the + compose bridge network is private. For production, point `DATABASE_URL` at a + TLS-enabled database, e.g. + `postgres://user:pass@db.example.com:5432/ctcloud?sslmode=verify-full`. + pgx handles TLS natively; no extra flags are needed. +* **Metrics** — bound to loopback. There is no auth, no debug endpoints and no + high-cardinality labels (never hostnames). +* **Resource limits** — `mem_limit`/`cpus`/`pids_limit` are set in compose so + overload is bounded by the queue sizes rather than by the OOM killer. +* **Authorization** — leave `ACTIVE_PROBING=false` unless you are authorised to + probe the discovered infrastructure. + +## Database schema + +* `cloud_assets` — one row per `(provider, service, provider_hostname)`, with + `hostname` (original CT hostname), `resource_identifier`, `region`, + `resource_specific`, `public_semantic`, `exposure_state`, `first_seen`, + `last_seen`. +* `observations` — append-only observations linking an asset to a + `source_hostname`, `source_type` (`direct_ct`/`cname`), CNAME chain and + A/AAAA records. +* `ct_checkpoints` — per-log CT resume state (JSON-encoded certspotter + `LogState`; Merkle positions, no certificates). +* `ct_sths` — downloaded, unverified signed tree heads needed for safe resume. + +Schema is applied idempotently at startup from embedded SQL in `migrations/`. +No certificate contents are ever stored. + +## Performance characteristics + +Optimised in this order: I/O off the hot path, then serialization, then process +boundaries, then allocations, then cheap suffix/label matching, then bounded +pools, then DNS caching, dedup and batched writes. + +The classifier: + +* no regexp, JSON, reflection, DNS, HTTP, filesystem, DB or logging; +* no `strings.Split`, no `fmt.Sprintf`; +* lowercases only when an uppercase byte is present; +* parses labels by byte offsets, returning substrings (zero copy); +* fixed stack arrays, no unbounded maps or goroutines. + +### Benchmark results (Apple M4, Go 1.27) + +```text +BenchmarkClassifyAWS-10 60.03 ns/op 0 B/op 0 allocs/op +BenchmarkClassifyAzure-10 44.93 ns/op 0 B/op 0 allocs/op +BenchmarkRejectNormalDomain-10 30.28 ns/op 0 B/op 0 allocs/op +BenchmarkClassifyMixed-10 369.50 ns/op 0 B/op 0 allocs/op +``` + +The whole in-process sink path is allocation-free too: + +```text +BenchmarkHandleHostNonMatch-10 115.6 ns/op 0 B/op 0 allocs/op +BenchmarkHandleHostDirectMatch-10 155.7 ns/op 0 B/op 0 allocs/op +``` + +Run them yourself: + +```sh +go test ./internal/classify/ ./internal/pipeline/ -run '^$' -bench . -benchmem +``` + +`BenchmarkClassifyMixed` and `TestHandleHostNoAllocs` fail unless the result is +exactly 0 allocs/op, so the target is enforced by the test suite. + +### DNS load balancing + +DNS lookups are round-robined across `DNS_SERVERS`. Each attempt is bounded +(2s) and fails over to the next server on timeout/refused errors, so one dead +resolver does not stall lookups. Definitive answers (NXDOMAIN) are returned +immediately rather than retried on every server, so negative lookups do not +multiply DNS traffic. + +## Testing + +```sh +go test ./... # unit + table-driven provider tests +go test -race ./... # race detector +go vet ./... +``` + +Integration tests for PostgreSQL run only when `DATABASE_URL` is set: + +```sh +DATABASE_URL=postgres://ctcloud:ctcloud@localhost:5432/ctcloud go test ./internal/store/ +``` + +## Limitations + +* `net.Resolver.LookupCNAME` returns the final canonical name, so intermediate + CNAME hops are not observed (the final target is still classified). Switch to + `miekg/dns` if the full chain is required. +* Optional reachability checks are off by default; when enabled they perform a + single non-destructive HTTPS `GET /` only — no authentication, downloads, + mutation or database queries. +* OCI generic `oraclecloud.com` endpoints are recognised but not stored, since + the bucket name lives in the URL path rather than the hostname. +* Custom R2 (and other) domains are only discovered through DNS/CNAME + enrichment, and only if the CT hostname itself appears in a certificate. diff --git a/cmd/ctcloud/main.go b/cmd/ctcloud/main.go new file mode 100644 index 0000000..3b723e7 --- /dev/null +++ b/cmd/ctcloud/main.go @@ -0,0 +1,234 @@ +// Command ctcloud continuously consumes Certificate Transparency logs through +// embedded CertSpotter internals, classifies cloud/database/storage resources +// in-process and persists only useful, enriched findings to PostgreSQL. +package main + +import ( + "context" + "errors" + "log/slog" + "net/http" + "os" + "os/signal" + "strings" + "sync/atomic" + "syscall" + "time" + + "software.sslmate.com/src/certspotter/monitor" + + "tea.karlcom.de/karl/bucketeer/internal/config" + "tea.karlcom.de/karl/bucketeer/internal/dns" + "tea.karlcom.de/karl/bucketeer/internal/metrics" + "tea.karlcom.de/karl/bucketeer/internal/model" + "tea.karlcom.de/karl/bucketeer/internal/pipeline" + "tea.karlcom.de/karl/bucketeer/internal/probe" + "tea.karlcom.de/karl/bucketeer/internal/store" +) + +func main() { + if err := run(); err != nil { + slog.Error("fatal", "error", err) + os.Exit(1) + } +} + +func levelFromString(s string) slog.Level { + switch s { + case "debug": + return slog.LevelDebug + case "warn": + return slog.LevelWarn + case "error": + return slog.LevelError + default: + return slog.LevelInfo + } +} + +func run() error { + cfg, err := config.Load() + if err != nil { + return err + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: levelFromString(cfg.LogLevel)}))) + if strings.Contains(cfg.DatabaseURL, "sslmode=disable") { + slog.Warn("DATABASE_URL has sslmode=disable; only use this on a private network") + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + m := metrics.New() + + st, err := store.Open(ctx, cfg.DatabaseURL, cfg.DBMaxConns) + if err != nil { + return err + } + defer st.Close() + st.SetMetrics(m) + if err := st.Migrate(ctx); err != nil { + return err + } + slog.Info("database ready") + + enricher := &dns.Enricher{ + Resolver: dns.NewServerPool(cfg.DNSServers), + Cache: dns.NewTTLCache[dns.Result](cfg.DNSCacheSize, 64), + PositiveTTL: cfg.DNSPositiveTTL, + NegativeTTL: cfg.DNSNegativeTTL, + Timeout: cfg.DNSTimeout, + MaxDepth: cfg.DNSMaxCNAMEdepth, + Metrics: m, + } + + var prober *probe.Prober + if cfg.ActiveProbing { + prober = probe.New(probe.Config{ + Workers: cfg.ProbeWorkers, + Timeout: cfg.ProbeTimeout, + Metrics: m, + Update: func(uctx context.Context, c model.Candidate, state model.ExposureState) { + _ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state) + }, + }) + prober.Start(ctx) + slog.Info("active probing enabled") + } + + write := func(wctx context.Context, batch []pipeline.Finding) error { + err := st.WriteFindings(wctx, batch) + if err == nil && prober != nil { + for i := range batch { + prober.Submit(batch[i].Candidate) + } + } + return err + } + + p := pipeline.New(pipeline.Config{ + DirectQueueSize: cfg.DirectQueueSize, + DNSQueueSize: cfg.DNSQueueSize, + DBQueueSize: cfg.DBQueueSize, + DNSWorkers: cfg.DNSWorkers, + BatchSize: cfg.DBBatchSize, + FlushInterval: cfg.DBFlushInterval, + DedupTTL: cfg.DedupeTTL, + Enricher: enricher, + Write: write, + Metrics: m, + }) + p.Start(ctx) + + var monitorReady atomic.Bool + monitorCfg := &monitor.Config{ + LogListSource: cfg.LogListSource, + State: st, + StartAtEnd: cfg.StartAtEnd, + Verbose: cfg.LogLevel == "debug", + HealthCheckInterval: 24 * time.Hour, + IdentifierSink: p, + } + + monitorDone := make(chan error, 1) + go func() { + monitorReady.Store(true) + monitorDone <- monitor.Run(ctx, monitorCfg) + }() + + httpSrv := startHTTP(cfg.MetricsAddr, m, st, &monitorReady) + go logStats(ctx, m, cfg.StatsInterval) + + slog.Info("ctcloud started") + err = <-monitorDone + if err != nil && !errors.Is(err, context.Canceled) { + slog.Error("ct monitor stopped", "error", err) + } + + // Graceful shutdown: stop intake, drain in-flight work, flush, then exit. + slog.Info("shutting down") + shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout) + defer cancel() + + drained := make(chan struct{}) + go func() { + p.Close() + if prober != nil { + prober.Wait() + } + close(drained) + }() + select { + case <-drained: + case <-shutdownCtx.Done(): + slog.Warn("shutdown timed out before pipeline drained") + } + + if err := httpSrv.Shutdown(shutdownCtx); err != nil { + slog.Warn("http shutdown", "error", err) + } + slog.Info("stopped", "certificates", m.CTCertificates.Load(), "candidates", m.Candidates.Load(), "rows", m.DBRows.Load()) + return nil +} + +func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ready *atomic.Bool) *http.Server { + mux := http.NewServeMux() + mux.Handle("/metrics", m.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if !ready.Load() { + http.Error(w, "ct monitor not initialized", http.StatusServiceUnavailable) + return + } + if err := st.Ping(ctx); err != nil { + http.Error(w, "database unreachable", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ready\n")) + }) + srv := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + slog.Error("metrics server", "error", err) + } + }() + return srv +} + +func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + slog.Info("stats", + "certificates", m.CTCertificates.Load(), + "identifiers", m.CTIdentifiers.Load(), + "direct_matches", m.DirectMatches.Load(), + "cname_matches", m.CNAMEMatches.Load(), + "candidates", m.Candidates.Load(), + "dns_queries", m.DNSQueries.Load(), + "dns_cache_hits", m.DNSCacheHits.Load(), + "dns_errors", m.DNSErrors.Load(), + "db_rows", m.DBRows.Load(), + "db_errors", m.DBErrors.Load(), + "queue_drops", m.QueueDrops.Load(), + ) + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..10a3e17 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,66 @@ +# Local / single-host deployment. +# +# Secrets come from .env (gitignored). Copy .env.example to .env and set a +# strong POSTGRES_PASSWORD plus a matching DATABASE_URL before `docker compose +# up`. For a production database, point DATABASE_URL at a TLS-enabled endpoint, +# e.g. postgres://user:pass@db.example.com:5432/ctcloud?sslmode=verify-full +# +# The compose bridge network is private; sslmode=disable is only acceptable here +# because the database is not reachable from outside the stack. +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:-ctcloud} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + POSTGRES_DB: ${POSTGRES_DB:-ctcloud} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-ctcloud} -d ${POSTGRES_DB:-ctcloud}"] + interval: 5s + timeout: 5s + retries: 12 + mem_limit: 1g + cpus: 2.0 + pids_limit: 256 + logging: + driver: json-file + options: { max-size: "10m", max-file: "3" } + restart: unless-stopped + + ctcloud: + build: . + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env} + METRICS_ADDR: ":9090" + LOG_LEVEL: ${LOG_LEVEL:-info} + DNS_SERVERS: ${DNS_SERVERS:-1.1.1.1:53,8.8.8.8:53,1.0.0.1:53} + DNS_WORKERS: ${DNS_WORKERS:-16} + ACTIVE_PROBING: ${ACTIVE_PROBING:-false} + # START_AT_END: "true" # skip historical certificates on first run + # Metrics/health are bound to loopback only: scrape from the host or a + # sidecar, never expose them publicly (there is no auth on this listener). + ports: + - "127.0.0.1:9090:9090" + # No application volume: the container is stateless apart from PostgreSQL. + read_only: true + tmpfs: + - /tmp:size=16m,noexec,nosuid + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + mem_limit: 1g + cpus: 2.0 + pids_limit: 512 + logging: + driver: json-file + options: { max-size: "10m", max-file: "3" } + restart: unless-stopped + +volumes: + pgdata: diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b1b009a --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module tea.karlcom.de/karl/bucketeer + +go 1.25.0 + +replace software.sslmate.com/src/certspotter => ./third_party/certspotter + +require ( + github.com/jackc/pgx/v5 v5.11.0 + software.sslmate.com/src/certspotter v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f6adc7f --- /dev/null +++ b/go.sum @@ -0,0 +1,32 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/classify/aws.go b/internal/classify/aws.go new file mode 100644 index 0000000..daf98b3 --- /dev/null +++ b/internal/classify/aws.go @@ -0,0 +1,148 @@ +package classify + +import ( + "strings" + + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +func aws(p model.Provider, s model.Service, sem model.PublicSemantic, resource, region string) model.Match { + return model.Match{ + Provider: p, + Service: s, + PublicSemantic: sem, + ResourceSpecific: true, + Region: region, + Resource: resource, + } +} + +func classifyAWS(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "amazonaws" { + return model.Match{}, false + } + if m, ok := classifyAWSS3(s); ok { + return m, true + } + + // Service-after-region forms: ...amazonaws.com + if n >= 5 { + switch s.label(n - 3) { + case "rds": + return awsResourceAfterRegion(s, model.ServiceRDS) + case "docdb": + return awsResourceAfterRegion(s, model.ServiceDocumentDB) + case "neptune": + return awsResourceAfterRegion(s, model.ServiceNeptune) + case "redshift", "redshift-serverless": + return awsResourceAfterRegion(s, model.ServiceRedshift) + case "es": + first := s.label(0) + switch { + case strings.HasPrefix(first, "search-"): + return aws(model.ProviderAWS, model.ServiceOpenSearch, + model.PublicSemanticPublicEndpoint, first, s.label(n-4)), true + case strings.HasPrefix(first, "vpc-"): + return aws(model.ProviderAWS, model.ServiceOpenSearch, + model.PublicSemanticVPCEndpoint, first, s.label(n-4)), true + } + case "cache": + return aws(model.ProviderAWS, model.ServiceElastiCache, + model.PublicSemanticNone, s.label(0), s.label(n-4)), true + } + } + + // Service-before-region form: .memorydb..amazonaws.com + if n >= 5 && s.label(n-4) == "memorydb" { + return aws(model.ProviderAWS, model.ServiceMemoryDB, + model.PublicSemanticNone, s.label(0), s.label(n-3)), true + } + return model.Match{}, false +} + +// awsResourceAfterRegion matches ...amazonaws.com +// and rejects the bare ..amazonaws.com service endpoint. +func awsResourceAfterRegion(s *scanner, service model.Service) (model.Match, bool) { + n := s.n + if n < 5 { + return model.Match{}, false + } + resource := s.label(0) + if resource == s.label(n-3) { + return model.Match{}, false + } + return aws(model.ProviderAWS, service, model.PublicSemanticNone, resource, s.label(n-4)), true +} + +func classifyAWSS3(s *scanner) (model.Match, bool) { + n := s.n + + // Access points: .s3-accesspoint[-fips][.dualstack]..amazonaws.com + if n >= 5 { + svcIdx := -1 + switch { + case s.label(n-4) == "s3-accesspoint" || s.label(n-4) == "s3-accesspoint-fips": + svcIdx = n - 4 + case n >= 6 && s.label(n-4) == "dualstack" && s.label(n-5) == "s3-accesspoint": + svcIdx = n - 5 + } + if svcIdx > 0 { + res := s.label(0) + if res != "s3-accesspoint" && res != "s3-accesspoint-fips" { + return aws(model.ProviderAWS, model.ServiceS3, + model.PublicSemanticNone, res, s.label(n-3)), true + } + } + } + + // Multi-region access point: .mrap.accesspoint.s3-global.amazonaws.com + if n >= 6 && s.label(n-5) == "mrap" && s.label(n-4) == "accesspoint" && + s.label(n-3) == "s3-global" { + return aws(model.ProviderAWS, model.ServiceS3, + model.PublicSemanticNone, s.label(0), ""), true + } + + // S3 Express: .s3express-..amazonaws.com + if n >= 5 && strings.HasPrefix(s.label(n-4), "s3express-") { + return aws(model.ProviderAWS, model.ServiceS3, + model.PublicSemanticNone, s.label(0), s.label(n-3)), true + } + + // S3 website: .s3-website..amazonaws.com + if n >= 5 && s.label(n-4) == "s3-website" { + return aws(model.ProviderAWS, model.ServiceS3, + model.PublicSemanticNone, s.label(0), s.label(n-3)), true + } + // S3 website (region in the service label): .s3-website-.amazonaws.com + if n >= 4 && strings.HasPrefix(s.label(n-3), "s3-website-") { + return aws(model.ProviderAWS, model.ServiceS3, + model.PublicSemanticNone, s.label(0), ""), true + } + + // Plain S3 without region: .s3.amazonaws.com + if n == 4 && s.label(n-3) == "s3" { + res := s.label(0) + if res == "s3" { + return model.Match{}, false + } + return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, ""), true + } + // Plain S3 with region: .s3..amazonaws.com + if n >= 5 && s.label(n-4) == "s3" { + res := s.label(0) + if res == "s3" { + return model.Match{}, false + } + return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true + } + // Dualstack S3: .s3.dualstack..amazonaws.com + if n >= 6 && s.label(n-5) == "s3" && s.label(n-4) == "dualstack" { + res := s.label(0) + if res == "s3" { + return model.Match{}, false + } + return aws(model.ProviderAWS, model.ServiceS3, model.PublicSemanticNone, res, s.label(n-3)), true + } + return model.Match{}, false +} diff --git a/internal/classify/azure.go b/internal/classify/azure.go new file mode 100644 index 0000000..ece3ae8 --- /dev/null +++ b/internal/classify/azure.go @@ -0,0 +1,113 @@ +package classify + +import "tea.karlcom.de/karl/bucketeer/internal/model" + +func azure(service model.Service, resource, region string) model.Match { + return model.Match{ + Provider: model.ProviderAzure, + Service: service, + ResourceSpecific: true, + Region: region, + Resource: resource, + } +} + +// classifyAzureCore handles *.blob|dfs|file|queue|table|web.core.windows.net. +func classifyAzureCore(s *scanner) (model.Match, bool) { + n := s.n + if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "core" { + return model.Match{}, false + } + svc := s.label(n - 4) + resource := s.label(0) + if resource == "" { + return model.Match{}, false + } + switch svc { + case "blob", "dfs": + return azure(model.ServiceBlobStorage, resource, ""), true + case "file": + return azure(model.ServiceFileStorage, resource, ""), true + case "queue": + return azure(model.ServiceQueue, resource, ""), true + case "table": + return azure(model.ServiceTableStorage, resource, ""), true + case "web": + return azure(model.ServiceWebApp, resource, ""), true + } + return model.Match{}, false +} + +// classifyAzureDatabase handles .database.windows.net. +func classifyAzureDatabase(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "database" { + return model.Match{}, false + } + return azure(model.ServiceDatabase, s.label(0), ""), true +} + +// classifyAzureFlexibleServer handles .{postgres,mysql}.database.azure.com. +func classifyAzureFlexibleServer(s *scanner) (model.Match, bool) { + n := s.n + if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "database" { + return model.Match{}, false + } + switch s.label(n - 4) { + case "postgres": + return azure(model.ServicePostgres, s.label(0), ""), true + case "mysql": + return azure(model.ServiceMySQL, s.label(0), ""), true + } + return model.Match{}, false +} + +// classifyAzureDocuments handles .documents.azure.com (Cosmos DB). +func classifyAzureDocuments(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "documents" { + return model.Match{}, false + } + return azure(model.ServiceCosmosDB, s.label(0), ""), true +} + +// classifyAzureCosmos handles .{mongo,cassandra,gremlin}.cosmos.azure.com. +func classifyAzureCosmos(s *scanner) (model.Match, bool) { + n := s.n + if n < 5 || s.label(n-1) != "com" || s.label(n-2) != "azure" || s.label(n-3) != "cosmos" { + return model.Match{}, false + } + switch s.label(n - 4) { + case "mongo", "cassandra", "gremlin": + return azure(model.ServiceCosmosDB, s.label(0), ""), true + } + return model.Match{}, false +} + +// classifyAzureRedis handles .redis.cache.windows.net. +func classifyAzureRedis(s *scanner) (model.Match, bool) { + n := s.n + if n < 5 || s.label(n-1) != "net" || s.label(n-2) != "windows" || + s.label(n-3) != "cache" || s.label(n-4) != "redis" { + return model.Match{}, false + } + return azure(model.ServiceRedis, s.label(0), ""), true +} + +// classifyAzureManagedRedis handles .redis.azure.net. +func classifyAzureManagedRedis(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "azure" || s.label(n-3) != "redis" { + return model.Match{}, false + } + return azure(model.ServiceRedis, s.label(0), ""), true +} + +// classifyAzureSearch handles .search.windows.net. +func classifyAzureSearch(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "net" || s.label(n-2) != "windows" || s.label(n-3) != "search" { + return model.Match{}, false + } + return azure(model.ServiceSearch, s.label(0), ""), true +} diff --git a/internal/classify/classify.go b/internal/classify/classify.go new file mode 100644 index 0000000..4849acf --- /dev/null +++ b/internal/classify/classify.go @@ -0,0 +1,198 @@ +// Package classify maps a DNS hostname to a cloud/database/storage resource. +// +// The hot path is deliberately allocation-free: it never lowercases unless an +// uppercase byte is present, never splits the hostname into a []string, never +// uses regexp/JSON/reflection, and returns substrings of the input as the +// region/resource. See BenchmarkClassify* for the 0 allocs/op assertion. +package classify + +import ( + "strings" + + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +const maxLabels = 64 + +// scanner holds the byte offsets of each dot-separated label of a hostname. +// It is stack allocated by Classify and never heap allocated. +type scanner struct { + host string + n int + start [maxLabels]int32 +} + +func newScanner(host string) scanner { + var s scanner + s.host = host + start := int32(0) + for i := 0; i < len(host); i++ { + if host[i] == '.' { + if s.n >= maxLabels { + s.n = -1 + return s + } + s.start[s.n] = start + s.n++ + start = int32(i + 1) + } + } + if s.n >= maxLabels { + s.n = -1 + return s + } + s.start[s.n] = start + s.n++ + return s +} + +// label returns the i-th label (0 = leftmost). Callers must ensure i < n. +func (s *scanner) label(i int) string { + end := int32(len(s.host)) + if i+1 < s.n { + end = s.start[i+1] - 1 + } + return s.host[s.start[i]:end] +} + +func hasUpper(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= 'A' && s[i] <= 'Z' { + return true + } + } + return false +} + +func trimTrailingDots(s string) string { + for len(s) > 0 && s[len(s)-1] == '.' { + s = s[:len(s)-1] + } + return s +} + +func validHost(host string) bool { + if host == "" || len(host) > 253 { + return false + } + for i := 0; i < len(host); i++ { + c := host[i] + switch { + case c >= 'a' && c <= 'z': + case c >= '0' && c <= '9': + case c == '-' || c == '.' || c == '_': + default: + return false + } + } + return true +} + +// Classify returns the provider resource described by host, if any. +// The returned Match's string fields alias host. +func Classify(host string) (model.Match, bool) { + host = trimTrailingDots(host) + if host == "" || len(host) > 253 { + return model.Match{}, false + } + // Wildcard certificates never describe a concrete resource. + if host[0] == '*' { + return model.Match{}, false + } + if hasUpper(host) { + host = strings.ToLower(host) + } + if !validHost(host) { + return model.Match{}, false + } + + s := newScanner(host) + if s.n < 2 { + return model.Match{}, false + } + + switch { + case strings.HasSuffix(host, ".amazonaws.com"): + return classifyAWS(&s) + case strings.HasSuffix(host, ".core.windows.net"): + return classifyAzureCore(&s) + case strings.HasSuffix(host, ".database.windows.net"): + return classifyAzureDatabase(&s) + case strings.HasSuffix(host, ".database.azure.com"): + return classifyAzureFlexibleServer(&s) + case strings.HasSuffix(host, ".documents.azure.com"): + return classifyAzureDocuments(&s) + case strings.HasSuffix(host, ".cosmos.azure.com"): + return classifyAzureCosmos(&s) + case strings.HasSuffix(host, ".cache.windows.net"): + return classifyAzureRedis(&s) + case strings.HasSuffix(host, ".redis.azure.net"): + return classifyAzureManagedRedis(&s) + case strings.HasSuffix(host, ".search.windows.net"): + return classifyAzureSearch(&s) + case strings.HasSuffix(host, ".storage.googleapis.com"): + return classifyGCS(&s) + case strings.HasSuffix(host, ".firebaseio.com"): + return classifyFirebaseIO(&s) + case strings.HasSuffix(host, ".firebasedatabase.app"): + return classifyFirebaseDB(&s) + case strings.HasSuffix(host, ".appspot.com"): + return classifyAppspot(&s) + case strings.HasSuffix(host, ".r2.cloudflarestorage.com"): + return classifyR2(&s) + case strings.HasSuffix(host, ".r2.dev"): + return classifyR2Dev(&s) + case strings.HasSuffix(host, ".digitaloceanspaces.com"): + return classifyDigitalOceanSpaces(&s) + case strings.HasSuffix(host, ".ondigitalocean.com"): + return classifyDigitalOceanDB(&s) + case strings.HasSuffix(host, ".supabase.co"): + return classifySupabase(&s) + case strings.HasSuffix(host, ".supabase.com"): + return classifySupabasePooler(&s) + case strings.HasSuffix(host, ".customer-oci.com"): + return classifyOCIResource(&s) + case strings.HasSuffix(host, ".oraclecloud.com"): + return classifyOCIGeneric(&s) + } + return model.Match{}, false +} + +// ShouldEnrich reports whether a non-matching hostname is worth a DNS/CNAME +// lookup. It rejects wildcards, the certspotter unparsable placeholder, single +// labels and obvious provider service apexes. +func ShouldEnrich(host string) bool { + host = trimTrailingDots(host) + if len(host) < 3 || host[0] == '*' { + return false + } + if host[0] == '<' { // certspotter's "" placeholder + return false + } + if !strings.Contains(host, ".") { + return false + } + if hasUpper(host) { + host = strings.ToLower(host) + } + // Provider service apexes never yield a customer resource via CNAME. + switch { + case host == "amazonaws.com", strings.HasSuffix(host, ".amazonaws.com"): + return false + case host == "core.windows.net", strings.HasSuffix(host, ".windows.net"): + return false + case host == "azure.com", strings.HasSuffix(host, ".azure.com"): + return false + case host == "googleapis.com", strings.HasSuffix(host, ".googleapis.com"): + return false + case host == "cloudflarestorage.com", host == "r2.dev": + return false + case strings.HasSuffix(host, ".digitaloceanspaces.com"): + return false + case strings.HasSuffix(host, ".supabase.co"), strings.HasSuffix(host, ".supabase.com"): + return false + case strings.HasSuffix(host, ".oraclecloud.com"), strings.HasSuffix(host, ".customer-oci.com"): + return false + } + return true +} diff --git a/internal/classify/classify_test.go b/internal/classify/classify_test.go new file mode 100644 index 0000000..534d878 --- /dev/null +++ b/internal/classify/classify_test.go @@ -0,0 +1,243 @@ +package classify + +import ( + "strings" + "testing" + + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +type want struct { + provider model.Provider + service model.Service + resource string + region string + resourceSpecific bool + semantic model.PublicSemantic +} + +func run(t *testing.T, host string, w *want) { + t.Helper() + m, ok := Classify(host) + if w == nil { + if ok { + t.Fatalf("Classify(%q) matched %+v, want no match", host, m) + } + return + } + if !ok { + t.Fatalf("Classify(%q) did not match, want %+v", host, *w) + } + if m.Provider != w.provider || m.Service != w.service { + t.Fatalf("Classify(%q) = %s/%s, want %s/%s", host, m.Provider, m.Service, w.provider, w.service) + } + if w.resource != "" && m.Resource != w.resource { + t.Fatalf("Classify(%q) resource = %q, want %q", host, m.Resource, w.resource) + } + if w.region != "" && m.Region != w.region { + t.Fatalf("Classify(%q) region = %q, want %q", host, m.Region, w.region) + } + if m.ResourceSpecific != w.resourceSpecific { + t.Fatalf("Classify(%q) resourceSpecific = %v, want %v", host, m.ResourceSpecific, w.resourceSpecific) + } + if w.semantic != 0 && m.PublicSemantic != w.semantic { + t.Fatalf("Classify(%q) semantic = %s, want %s", host, m.PublicSemantic, w.semantic) + } +} + +func TestClassifyMatches(t *testing.T) { + cases := []struct { + host string + w want + }{ + // AWS S3 + {"bucket.s3.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}}, + {"bucket.s3.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}}, + {"bucket.s3.dualstack.eu-central-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}}, + {"bucket.s3-website.eu-west-2.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-2", true, 0}}, + {"bucket.s3-website-us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}}, + {"ap-123456.s3-accesspoint.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "ap-123456", "us-east-1", true, 0}}, + {"x.s3-accesspoint.dualstack.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}}, + {"x.s3-accesspoint-fips.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "us-east-1", true, 0}}, + {"x.mrap.accesspoint.s3-global.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "x", "", true, 0}}, + {"bucket.s3express-zone1.eu-west-1.amazonaws.com", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-west-1", true, 0}}, + // AWS databases + {"mydb.cluster-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}}, + {"mydb.cluster-ro-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}}, + {"mydb.proxy-abc.us-east-1.rds.amazonaws.com", want{model.ProviderAWS, model.ServiceRDS, "mydb", "us-east-1", true, 0}}, + {"mydb.cluster-abc.us-east-1.docdb.amazonaws.com", want{model.ProviderAWS, model.ServiceDocumentDB, "mydb", "us-east-1", true, 0}}, + {"cluster.id.eu-central-1.redshift.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}}, + {"cluster.id.eu-central-1.redshift-serverless.amazonaws.com", want{model.ProviderAWS, model.ServiceRedshift, "cluster", "eu-central-1", true, 0}}, + {"search-example-abc.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "search-example-abc", "eu-west-1", true, model.PublicSemanticPublicEndpoint}}, + {"vpc-example.eu-west-1.es.amazonaws.com", want{model.ProviderAWS, model.ServiceOpenSearch, "vpc-example", "eu-west-1", true, model.PublicSemanticVPCEndpoint}}, + {"mydb.cluster-abc.us-east-1.neptune.amazonaws.com", want{model.ProviderAWS, model.ServiceNeptune, "mydb", "us-east-1", true, 0}}, + {"clustercfg.my.xxxx.memorydb.us-east-1.amazonaws.com", want{model.ProviderAWS, model.ServiceMemoryDB, "clustercfg", "us-east-1", true, 0}}, + {"mycluster.abc.0001.use1.cache.amazonaws.com", want{model.ProviderAWS, model.ServiceElastiCache, "mycluster", "use1", true, 0}}, + // Azure + {"myacct.blob.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}}, + {"myacct.dfs.core.windows.net", want{model.ProviderAzure, model.ServiceBlobStorage, "myacct", "", true, 0}}, + {"myacct.file.core.windows.net", want{model.ProviderAzure, model.ServiceFileStorage, "myacct", "", true, 0}}, + {"myacct.queue.core.windows.net", want{model.ProviderAzure, model.ServiceQueue, "myacct", "", true, 0}}, + {"myacct.table.core.windows.net", want{model.ProviderAzure, model.ServiceTableStorage, "myacct", "", true, 0}}, + {"myacct.web.core.windows.net", want{model.ProviderAzure, model.ServiceWebApp, "myacct", "", true, 0}}, + {"myserver.database.windows.net", want{model.ProviderAzure, model.ServiceDatabase, "myserver", "", true, 0}}, + {"myserver.postgres.database.azure.com", want{model.ProviderAzure, model.ServicePostgres, "myserver", "", true, 0}}, + {"myserver.mysql.database.azure.com", want{model.ProviderAzure, model.ServiceMySQL, "myserver", "", true, 0}}, + {"mycosmos.documents.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}}, + {"mycosmos.mongo.cosmos.azure.com", want{model.ProviderAzure, model.ServiceCosmosDB, "mycosmos", "", true, 0}}, + {"myredis.redis.cache.windows.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}}, + {"myredis.redis.azure.net", want{model.ProviderAzure, model.ServiceRedis, "myredis", "", true, 0}}, + {"mysearch.search.windows.net", want{model.ProviderAzure, model.ServiceSearch, "mysearch", "", true, 0}}, + // GCP + {"bucket.storage.googleapis.com", want{model.ProviderGCP, model.ServiceObjectStorage, "bucket", "", true, 0}}, + {"project.firebaseio.com", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "", true, 0}}, + {"project.europe-west1.firebasedatabase.app", want{model.ProviderGCP, model.ServiceFirebaseDB, "project", "europe-west1", true, 0}}, + {"myproject.appspot.com", want{model.ProviderGCP, model.ServiceWebApp, "myproject", "", true, 0}}, + // Cloudflare + {"acct.r2.cloudflarestorage.com", want{model.ProviderCloudflare, model.ServiceObjectStorage, "acct", "", true, 0}}, + {"mybucket.r2.dev", want{model.ProviderCloudflare, model.ServiceObjectStorage, "mybucket", "", true, model.PublicSemanticPublicURL}}, + // DigitalOcean + {"bucket.fra1.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}}, + {"bucket.fra1.cdn.digitaloceanspaces.com", want{model.ProviderDigitalOcean, model.ServiceObjectStorage, "bucket", "fra1", true, 0}}, + {"mydb.db.ondigitalocean.com", want{model.ProviderDigitalOcean, model.ServiceDatabase, "mydb", "", true, 0}}, + // Supabase + {"abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServiceDatabase, "abcdefgh", "", true, 0}}, + {"db.abcdefgh.supabase.co", want{model.ProviderSupabase, model.ServicePostgres, "abcdefgh", "", true, 0}}, + {"aws-0-us-east-1.pooler.supabase.com", want{model.ProviderSupabase, model.ServicePostgres, "aws-0-us-east-1", "", true, 0}}, + // OCI + {"myns.objectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}}, + {"myns.swiftobjectstorage.us-ashburn-1.oci.customer-oci.com", want{model.ProviderOracle, model.ServiceObjectStorage, "myns", "us-ashburn-1", true, 0}}, + {"objectstorage.us-ashburn-1.oraclecloud.com", want{model.ProviderOracle, model.ServiceObjectStorage, "", "us-ashburn-1", false, 0}}, + // normalization + {"BUCKET.S3.EU-CENTRAL-1.AMAZONAWS.COM", want{model.ProviderAWS, model.ServiceS3, "bucket", "eu-central-1", true, 0}}, + {"bucket.s3.amazonaws.com.", want{model.ProviderAWS, model.ServiceS3, "bucket", "", true, 0}}, + } + for _, c := range cases { + w := c.w + run(t, c.host, &w) + } +} + +func TestClassifyRejects(t *testing.T) { + rejects := []string{ + // AWS generic endpoints / non-resources + "amazonaws.com", + "s3.amazonaws.com", + "s3.eu-central-1.amazonaws.com", + "s3.dualstack.eu-central-1.amazonaws.com", + "s3-accesspoint.us-east-1.amazonaws.com", + "dynamodb.eu-central-1.amazonaws.com", + "redshift.eu-central-1.amazonaws.com", + "rds.us-east-1.amazonaws.com", + "docdb.us-east-1.amazonaws.com", + "neptune.us-east-1.amazonaws.com", + "es.us-east-1.amazonaws.com", + "memorydb.us-east-1.amazonaws.com", + "cache.amazonaws.com", + "use1.cache.amazonaws.com", + "s3-global.amazonaws.com", + // Azure bare service endpoints + "blob.core.windows.net", + "database.windows.net", + "postgres.database.azure.com", + "documents.azure.com", + "mongo.cosmos.azure.com", + "redis.cache.windows.net", + "redis.azure.net", + "search.windows.net", + // GCP bare service endpoints + "storage.googleapis.com", + "firebaseio.com", + "firebasedatabase.app", + "appspot.com", + // Cloudflare + "r2.cloudflarestorage.com", + "r2.dev", + // DigitalOcean + "fra1.digitaloceanspaces.com", + "db.ondigitalocean.com", + // Supabase + "supabase.co", + "pooler.supabase.com", + // OCI + "oraclecloud.com", + "customer-oci.com", + // wildcards + "*.example.com", + "*.blob.core.windows.net", + "*.s3.amazonaws.com", + // normal hostnames + "www.example.com", + "foo.bar.example.org", + "localhost", + "", + "-", + // invalid / too long + "foo bar.com", + strings.Repeat("a.", 130) + "com", + } + for _, host := range rejects { + run(t, host, nil) + } +} + +func TestShouldEnrich(t *testing.T) { + yes := []string{"files.example.com", "db.example.com", "a.b.c.example.org"} + no := []string{"*.example.com", "s3.amazonaws.com", "foo.s3.amazonaws.com", "storage.googleapis.com", "", ""} + for _, h := range yes { + if !ShouldEnrich(h) { + t.Errorf("ShouldEnrich(%q) = false, want true", h) + } + } + for _, h := range no { + if ShouldEnrich(h) { + t.Errorf("ShouldEnrich(%q) = true, want false", h) + } + } +} + +var benchInputs = []string{ + "www.example.com", + "foo.bar.example.org", + "bucket.s3.eu-central-1.amazonaws.com", + "foo.blob.core.windows.net", + "db.example.supabase.co", + "project.firebaseio.com", + "search-example-abc.eu-west-1.es.amazonaws.com", +} + +func BenchmarkClassifyAWS(b *testing.B) { + for i := 0; i < b.N; i++ { + Classify("bucket.s3.eu-central-1.amazonaws.com") + } +} + +func BenchmarkClassifyAzure(b *testing.B) { + for i := 0; i < b.N; i++ { + Classify("foo.blob.core.windows.net") + } +} + +func BenchmarkRejectNormalDomain(b *testing.B) { + for i := 0; i < b.N; i++ { + Classify("www.example.com") + } +} + +func BenchmarkClassifyMixed(b *testing.B) { + allocs := testing.AllocsPerRun(1000, func() { + for _, host := range benchInputs { + Classify(host) + } + }) + if allocs != 0 { + b.Fatalf("Classify allocates %.2f allocs/op, want 0", allocs) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, host := range benchInputs { + Classify(host) + } + } +} diff --git a/internal/classify/cloudflare.go b/internal/classify/cloudflare.go new file mode 100644 index 0000000..38056e0 --- /dev/null +++ b/internal/classify/cloudflare.go @@ -0,0 +1,34 @@ +package classify + +import "tea.karlcom.de/karl/bucketeer/internal/model" + +// classifyR2 handles .r2.cloudflarestorage.com. +func classifyR2(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "cloudflarestorage" || s.label(n-3) != "r2" { + return model.Match{}, false + } + return model.Match{ + Provider: model.ProviderCloudflare, + Service: model.ServiceObjectStorage, + ResourceSpecific: true, + Resource: s.label(0), + }, true +} + +// classifyR2Dev handles .r2.dev. The hostname form is explicitly a +// public development/public URL, which is why it carries PublicSemanticPublicURL +// while still not asserting that data is readable. +func classifyR2Dev(s *scanner) (model.Match, bool) { + n := s.n + if n < 3 || s.label(n-1) != "dev" || s.label(n-2) != "r2" { + return model.Match{}, false + } + return model.Match{ + Provider: model.ProviderCloudflare, + Service: model.ServiceObjectStorage, + ResourceSpecific: true, + PublicSemantic: model.PublicSemanticPublicURL, + Resource: s.label(0), + }, true +} diff --git a/internal/classify/digitalocean.go b/internal/classify/digitalocean.go new file mode 100644 index 0000000..556ae76 --- /dev/null +++ b/internal/classify/digitalocean.go @@ -0,0 +1,39 @@ +package classify + +import "tea.karlcom.de/karl/bucketeer/internal/model" + +// classifyDigitalOceanSpaces handles +// ..digitaloceanspaces.com and the cdn variant. +func classifyDigitalOceanSpaces(s *scanner) (model.Match, bool) { + n := s.n + if s.label(n-1) != "com" || s.label(n-2) != "digitaloceanspaces" { + return model.Match{}, false + } + // ..digitaloceanspaces.com (n==4) or the cdn variant (n==5) + if n != 4 && !(n == 5 && s.label(2) == "cdn") { + return model.Match{}, false + } + bucket := s.label(0) + region := s.label(1) + return model.Match{ + Provider: model.ProviderDigitalOcean, + Service: model.ServiceObjectStorage, + ResourceSpecific: true, + Region: region, + Resource: bucket, + }, true +} + +// classifyDigitalOceanDB handles .db.ondigitalocean.com. +func classifyDigitalOceanDB(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "ondigitalocean" || s.label(n-3) != "db" { + return model.Match{}, false + } + return model.Match{ + Provider: model.ProviderDigitalOcean, + Service: model.ServiceDatabase, + ResourceSpecific: true, + Resource: s.label(0), + }, true +} diff --git a/internal/classify/gcp.go b/internal/classify/gcp.go new file mode 100644 index 0000000..5fb457f --- /dev/null +++ b/internal/classify/gcp.go @@ -0,0 +1,49 @@ +package classify + +import "tea.karlcom.de/karl/bucketeer/internal/model" + +func gcp(service model.Service, resource, region string) model.Match { + return model.Match{ + Provider: model.ProviderGCP, + Service: service, + ResourceSpecific: true, + Region: region, + Resource: resource, + } +} + +// classifyGCS handles .storage.googleapis.com. +func classifyGCS(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "com" || s.label(n-2) != "googleapis" || s.label(n-3) != "storage" { + return model.Match{}, false + } + return gcp(model.ServiceObjectStorage, s.label(0), ""), true +} + +// classifyFirebaseIO handles .firebaseio.com. +func classifyFirebaseIO(s *scanner) (model.Match, bool) { + n := s.n + if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "firebaseio" { + return model.Match{}, false + } + return gcp(model.ServiceFirebaseDB, s.label(0), ""), true +} + +// classifyFirebaseDB handles ..firebasedatabase.app. +func classifyFirebaseDB(s *scanner) (model.Match, bool) { + n := s.n + if n < 4 || s.label(n-1) != "app" || s.label(n-2) != "firebasedatabase" { + return model.Match{}, false + } + return gcp(model.ServiceFirebaseDB, s.label(0), s.label(1)), true +} + +// classifyAppspot handles .appspot.com. +func classifyAppspot(s *scanner) (model.Match, bool) { + n := s.n + if n < 3 || s.label(n-1) != "com" || s.label(n-2) != "appspot" { + return model.Match{}, false + } + return gcp(model.ServiceWebApp, s.label(0), ""), true +} diff --git a/internal/classify/oracle.go b/internal/classify/oracle.go new file mode 100644 index 0000000..0d54b28 --- /dev/null +++ b/internal/classify/oracle.go @@ -0,0 +1,44 @@ +package classify + +import "tea.karlcom.de/karl/bucketeer/internal/model" + +// classifyOCIResource handles +// .objectstorage..oci.customer-oci.com and the swift variant. +// The namespace is in the hostname, so it is resource specific. +func classifyOCIResource(s *scanner) (model.Match, bool) { + n := s.n + if n < 6 || s.label(n-1) != "com" || s.label(n-2) != "customer-oci" || s.label(n-3) != "oci" { + return model.Match{}, false + } + svc := s.label(n - 5) + if svc != "objectstorage" && svc != "swiftobjectstorage" { + return model.Match{}, false + } + return model.Match{ + Provider: model.ProviderOracle, + Service: model.ServiceObjectStorage, + ResourceSpecific: true, + Region: s.label(n - 4), + Resource: s.label(0), + }, true +} + +// classifyOCIGeneric handles objectstorage..oraclecloud.com. The bucket +// name lives in the URL path rather than the hostname, so resource_specific is +// false and these normally do not become findings. +func classifyOCIGeneric(s *scanner) (model.Match, bool) { + n := s.n + if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "oraclecloud" { + return model.Match{}, false + } + svc := s.label(n - 4) + if svc != "objectstorage" && svc != "swiftobjectstorage" { + return model.Match{}, false + } + return model.Match{ + Provider: model.ProviderOracle, + Service: model.ServiceObjectStorage, + ResourceSpecific: false, + Region: s.label(n - 3), + }, true +} diff --git a/internal/classify/supabase.go b/internal/classify/supabase.go new file mode 100644 index 0000000..13ad59f --- /dev/null +++ b/internal/classify/supabase.go @@ -0,0 +1,45 @@ +package classify + +import "tea.karlcom.de/karl/bucketeer/internal/model" + +// classifySupabase handles .supabase.co and +// db..supabase.co (the latter classified as PostgreSQL). +func classifySupabase(s *scanner) (model.Match, bool) { + n := s.n + if s.label(n-1) != "co" || s.label(n-2) != "supabase" { + return model.Match{}, false + } + switch n { + case 3: + return model.Match{ + Provider: model.ProviderSupabase, + Service: model.ServiceDatabase, + ResourceSpecific: true, + Resource: s.label(0), + }, true + case 4: + if s.label(0) == "db" { + return model.Match{ + Provider: model.ProviderSupabase, + Service: model.ServicePostgres, + ResourceSpecific: true, + Resource: s.label(1), + }, true + } + } + return model.Match{}, false +} + +// classifySupabasePooler handles aws--.pooler.supabase.com. +func classifySupabasePooler(s *scanner) (model.Match, bool) { + n := s.n + if n != 4 || s.label(n-1) != "com" || s.label(n-2) != "supabase" || s.label(n-3) != "pooler" { + return model.Match{}, false + } + return model.Match{ + Provider: model.ProviderSupabase, + Service: model.ServicePostgres, + ResourceSpecific: true, + Resource: s.label(0), + }, true +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..07285a5 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,160 @@ +// Package config loads all ctcloud configuration from environment variables. +// There are no configuration files at runtime. +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Config struct { + DatabaseURL string + DBMaxConns int + + DNSServers []string + DNSWorkers int + DNSTimeout time.Duration + DNSCacheSize int + DNSMaxCNAMEdepth int + DNSPositiveTTL time.Duration + DNSNegativeTTL time.Duration + + DirectQueueSize int + DNSQueueSize int + DBQueueSize int + + DBBatchSize int + DBFlushInterval time.Duration + + ActiveProbing bool + ProbeWorkers int + ProbeTimeout time.Duration + + MetricsAddr string + LogLevel string + StatsInterval time.Duration + + LogListSource string + StartAtEnd bool + ShutdownTimeout time.Duration + DedupeTTL time.Duration +} + +// DefaultLogListSource is the public CT log list used by certspotter. +const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.json" + +// DefaultDNSServers are load-balanced across. Set DNS_SERVERS=system to use the +// host resolver instead, or provide your own comma-separated list. +var DefaultDNSServers = []string{"1.1.1.1:53", "8.8.8.8:53", "1.0.0.1:53"} + +func Load() (*Config, error) { + c := &Config{ + DatabaseURL: os.Getenv("DATABASE_URL"), + DBMaxConns: envInt("DB_MAX_CONNS", 8), + DNSServers: envList("DNS_SERVERS", DefaultDNSServers), + DNSWorkers: envInt("DNS_WORKERS", 16), + DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second), + DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<20), + DNSMaxCNAMEdepth: envInt("DNS_MAX_CNAME_DEPTH", 8), + DNSPositiveTTL: envDuration("DNS_POSITIVE_TTL", 10*time.Minute), + DNSNegativeTTL: envDuration("DNS_NEGATIVE_TTL", 1*time.Minute), + DirectQueueSize: envInt("DIRECT_QUEUE_SIZE", 8192), + DNSQueueSize: envInt("DNS_QUEUE_SIZE", 65536), + DBQueueSize: envInt("DB_QUEUE_SIZE", 8192), + DBBatchSize: envInt("DB_BATCH_SIZE", 500), + DBFlushInterval: envDuration("DB_FLUSH_INTERVAL", 2*time.Second), + ActiveProbing: envBool("ACTIVE_PROBING", false), + ProbeWorkers: envInt("PROBE_WORKERS", 4), + ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second), + MetricsAddr: envString("METRICS_ADDR", ":9090"), + LogLevel: envString("LOG_LEVEL", "info"), + StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second), + LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource), + StartAtEnd: envBool("START_AT_END", false), + ShutdownTimeout: envDuration("SHUTDOWN_TIMEOUT", 20*time.Second), + DedupeTTL: envDuration("DEDUP_TTL", time.Hour), + } + if err := c.validate(); err != nil { + return nil, err + } + return c, nil +} + +func (c *Config) validate() error { + if c.DatabaseURL == "" { + return fmt.Errorf("DATABASE_URL is required") + } + if c.DNSWorkers < 1 { + return fmt.Errorf("DNS_WORKERS must be >= 1") + } + if c.DNSMaxCNAMEdepth < 1 { + return fmt.Errorf("DNS_MAX_CNAME_DEPTH must be >= 1") + } + if c.DirectQueueSize < 1 || c.DNSQueueSize < 1 || c.DBQueueSize < 1 { + return fmt.Errorf("queue sizes must be >= 1") + } + if c.DBBatchSize < 1 { + return fmt.Errorf("DB_BATCH_SIZE must be >= 1") + } + switch c.LogLevel { + case "debug", "info", "warn", "error": + default: + return fmt.Errorf("LOG_LEVEL must be one of debug, info, warn, error") + } + return nil +} + +func envList(key string, def []string) []string { + v := os.Getenv(key) + if v == "" { + return def + } + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + if len(out) == 0 { + return def + } + return out +} + +func envString(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil { + return n + } + } + return def +} + +func envBool(key string, def bool) bool { + if v := os.Getenv(key); v != "" { + if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil { + return b + } + } + return def +} + +func envDuration(key string, def time.Duration) time.Duration { + if v := os.Getenv(key); v != "" { + if d, err := time.ParseDuration(strings.TrimSpace(v)); err == nil { + return d + } + } + return def +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..9145be6 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,53 @@ +package config + +import ( + "testing" + "time" +) + +func TestLoadDefaults(t *testing.T) { + t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud") + c, err := Load() + if err != nil { + t.Fatal(err) + } + if c.DatabaseURL != "postgres://localhost/ctcloud" { + t.Fatalf("database url = %q", c.DatabaseURL) + } + if c.DNSWorkers != 16 || c.DNSMaxCNAMEdepth != 8 { + t.Fatalf("unexpected DNS defaults: %+v", c) + } + if c.ActiveProbing { + t.Fatal("active probing must default to false") + } +} + +func TestLoadOverrides(t *testing.T) { + t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud") + t.Setenv("DNS_WORKERS", "4") + t.Setenv("ACTIVE_PROBING", "true") + t.Setenv("DB_FLUSH_INTERVAL", "250ms") + t.Setenv("STATS_INTERVAL", "30s") + c, err := Load() + if err != nil { + t.Fatal(err) + } + if c.DNSWorkers != 4 || !c.ActiveProbing { + t.Fatalf("overrides not applied: %+v", c) + } + if c.DBFlushInterval != 250*time.Millisecond || c.StatsInterval != 30*time.Second { + t.Fatalf("duration overrides not applied: %+v", c) + } +} + +func TestLoadInvalid(t *testing.T) { + t.Setenv("DATABASE_URL", "") + if _, err := Load(); err == nil { + t.Fatal("expected error without DATABASE_URL") + } + t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud") + t.Setenv("LOG_LEVEL", "verbose") + if _, err := Load(); err == nil { + t.Fatal("expected error for bad LOG_LEVEL") + } +} diff --git a/internal/dns/cache.go b/internal/dns/cache.go new file mode 100644 index 0000000..67e4c4b --- /dev/null +++ b/internal/dns/cache.go @@ -0,0 +1,99 @@ +package dns + +import ( + "sync" + "time" +) + +type cacheEntry[V any] struct { + val V + exp time.Time +} + +type shard[V any] struct { + mu sync.Mutex + m map[string]cacheEntry[V] +} + +// TTLCache is a sharded, memory-bounded, in-memory TTL cache. It is used for +// positive DNS results, negative DNS results, CNAME targets and hostname dedup. +// +// ponytail: once a shard is full, an arbitrary entry is evicted rather than the +// least-recently-used one. Swap in an LRU only if the hit rate actually matters. +type TTLCache[V any] struct { + shards []shard[V] + mask uint64 + maxShard int +} + +func NewTTLCache[V any](maxEntries, shards int) *TTLCache[V] { + if shards < 1 { + shards = 1 + } + n := 1 + for n < shards { + n <<= 1 + } + maxShard := maxEntries / n + if maxShard < 1 { + maxShard = 1 + } + c := &TTLCache[V]{mask: uint64(n - 1), maxShard: maxShard} + c.shards = make([]shard[V], n) + for i := range c.shards { + c.shards[i].m = make(map[string]cacheEntry[V], maxShard) + } + return c +} + +func hashString(s string) uint64 { + const prime = 1099511628211 + h := uint64(14695981039346656037) + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= prime + } + return h +} + +func (c *TTLCache[V]) Get(key string) (V, bool) { + s := &c.shards[hashString(key)&c.mask] + s.mu.Lock() + e, ok := s.m[key] + if ok && time.Now().Before(e.exp) { + s.mu.Unlock() + return e.val, true + } + if ok { + delete(s.m, key) + } + s.mu.Unlock() + var zero V + return zero, false +} + +func (c *TTLCache[V]) Set(key string, val V, ttl time.Duration) { + if ttl <= 0 { + return + } + s := &c.shards[hashString(key)&c.mask] + s.mu.Lock() + if len(s.m) >= c.maxShard { + for k := range s.m { + delete(s.m, k) + break + } + } + s.m[key] = cacheEntry[V]{val: val, exp: time.Now().Add(ttl)} + s.mu.Unlock() +} + +func (c *TTLCache[V]) Len() int { + total := 0 + for i := range c.shards { + c.shards[i].mu.Lock() + total += len(c.shards[i].m) + c.shards[i].mu.Unlock() + } + return total +} diff --git a/internal/dns/pool.go b/internal/dns/pool.go new file mode 100644 index 0000000..7c49107 --- /dev/null +++ b/internal/dns/pool.go @@ -0,0 +1,119 @@ +package dns + +import ( + "context" + "errors" + "net" + "strings" + "sync/atomic" + "time" +) + +// ServerPool load-balances DNS lookups across multiple resolvers using +// round-robin, and fails over to the next server on a timeout or server error. +// Definitive answers (NXDOMAIN) are returned immediately and are not retried on +// every server. +// defaultAttemptTimeout bounds each individual server attempt so a dead +// resolver costs a couple of seconds before failover, not the entire lookup +// budget. +const defaultAttemptTimeout = 2 * time.Second + +type ServerPool struct { + resolvers []Resolver + next atomic.Uint64 + attemptTimeout time.Duration +} + +func (p *ServerPool) withAttemptTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + t := p.attemptTimeout + if t <= 0 { + t = defaultAttemptTimeout + } + return context.WithTimeout(ctx, t) +} + +// NewServerPool builds a pool from "host", "host:port" or "system" entries. +// An empty list uses the system resolver. +func NewServerPool(servers []string) *ServerPool { + var resolvers []Resolver + for _, s := range servers { + s = strings.TrimSpace(s) + switch { + case s == "": + continue + case s == "system" || s == "local": + return &ServerPool{resolvers: []Resolver{&GoResolver{R: net.DefaultResolver}}} + default: + if !strings.Contains(s, ":") { + s += ":53" + } + resolvers = append(resolvers, &GoResolver{R: newServerResolver(s)}) + } + } + if len(resolvers) == 0 { + resolvers = append(resolvers, &GoResolver{R: net.DefaultResolver}) + } + return &ServerPool{resolvers: resolvers} +} + +func newServerResolver(server string) *net.Resolver { + return &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { + if network != "tcp" { + network = "udp" + } + d := net.Dialer{Timeout: 5 * time.Second} + return d.DialContext(ctx, network, server) + }, + } +} + +func (p *ServerPool) pick() int { + return int((p.next.Add(1) - 1) % uint64(len(p.resolvers))) +} + +// definitive reports whether err is a real DNS answer (NXDOMAIN) that should not +// be retried against other servers. +func definitive(err error) bool { + var dnsErr *net.DNSError + return errors.As(err, &dnsErr) && dnsErr.IsNotFound +} + +func (p *ServerPool) LookupCNAME(ctx context.Context, host string) (string, error) { + n := len(p.resolvers) + start := p.pick() + var last error + for i := 0; i < n; i++ { + actx, cancel := p.withAttemptTimeout(ctx) + cname, err := p.resolvers[(start+i)%n].LookupCNAME(actx, host) + cancel() + if err == nil { + return cname, nil + } + if definitive(err) { + return "", err + } + last = err + } + return "", last +} + +func (p *ServerPool) LookupIP(ctx context.Context, host string) ([]net.IP, error) { + n := len(p.resolvers) + start := p.pick() + var last error + for i := 0; i < n; i++ { + actx, cancel := p.withAttemptTimeout(ctx) + ips, err := p.resolvers[(start+i)%n].LookupIP(actx, host) + cancel() + if err == nil { + return ips, nil + } + if definitive(err) { + return nil, err + } + last = err + } + return nil, last +} diff --git a/internal/dns/pool_test.go b/internal/dns/pool_test.go new file mode 100644 index 0000000..4b2826c --- /dev/null +++ b/internal/dns/pool_test.go @@ -0,0 +1,68 @@ +package dns + +import ( + "context" + "net" + "sync/atomic" + "testing" +) + +type fakeLookup struct { + calls atomic.Int32 + cnameErr error + ipErr error +} + +func (f *fakeLookup) LookupCNAME(ctx context.Context, host string) (string, error) { + f.calls.Add(1) + if f.cnameErr != nil { + return "", f.cnameErr + } + return host, nil +} + +func (f *fakeLookup) LookupIP(ctx context.Context, host string) ([]net.IP, error) { + f.calls.Add(1) + if f.ipErr != nil { + return nil, f.ipErr + } + return []net.IP{net.ParseIP("1.2.3.4")}, nil +} + +func TestServerPoolRoundRobin(t *testing.T) { + a, b := &fakeLookup{}, &fakeLookup{} + p := &ServerPool{resolvers: []Resolver{a, b}} + for i := 0; i < 4; i++ { + if _, err := p.LookupIP(context.Background(), "example.com"); err != nil { + t.Fatal(err) + } + } + if a.calls.Load() != 2 || b.calls.Load() != 2 { + t.Fatalf("round robin calls: a=%d b=%d, want 2/2", a.calls.Load(), b.calls.Load()) + } +} + +func TestServerPoolFailover(t *testing.T) { + a := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}} + b := &fakeLookup{} + p := &ServerPool{resolvers: []Resolver{a, b}} + ips, err := p.LookupIP(context.Background(), "example.com") + if err != nil { + t.Fatal(err) + } + if len(ips) != 1 || a.calls.Load() != 1 { + t.Fatalf("expected failover to second server, a.calls=%d", a.calls.Load()) + } +} + +func TestServerPoolNoFailoverOnNXDOMAIN(t *testing.T) { + a := &fakeLookup{ipErr: &net.DNSError{Err: "no such host", IsNotFound: true}} + b := &fakeLookup{} + p := &ServerPool{resolvers: []Resolver{a, b}} + if _, err := p.LookupIP(context.Background(), "missing.example.com"); err == nil { + t.Fatal("expected error") + } + if b.calls.Load() != 0 { + t.Fatalf("NXDOMAIN must not fail over; b.calls=%d", b.calls.Load()) + } +} diff --git a/internal/dns/resolver.go b/internal/dns/resolver.go new file mode 100644 index 0000000..dfff3de --- /dev/null +++ b/internal/dns/resolver.go @@ -0,0 +1,131 @@ +// Package dns provides a bounded, cached DNS enricher that follows CNAME chains +// with loop detection. +package dns + +import ( + "context" + "errors" + "net" + "strings" + "time" + + "tea.karlcom.de/karl/bucketeer/internal/metrics" +) + +// ErrCNAMELoop is returned when a CNAME chain revisits a host. +var ErrCNAMELoop = errors.New("cname loop detected") + +// Resolver is the subset of DNS operations the enricher needs. +type Resolver interface { + LookupCNAME(ctx context.Context, host string) (string, error) + LookupIP(ctx context.Context, host string) ([]net.IP, error) +} + +// Result is a resolved hostname: its CNAME chain (chain[0] is the queried host) +// and any A/AAAA addresses. +type Result struct { + Chain []string + IPs []net.IP +} + +// GoResolver uses the standard library resolver. +// +// ponytail: net.Resolver.LookupCNAME returns the final canonical name, so the +// chain has at most two entries and intermediate CNAME hops are not observed. +// Swap in miekg/dns if the full chain is ever needed. +type GoResolver struct { + R *net.Resolver +} + +func NewGoResolver() *GoResolver { + return &GoResolver{R: net.DefaultResolver} +} + +func (g *GoResolver) LookupCNAME(ctx context.Context, host string) (string, error) { + return g.R.LookupCNAME(ctx, host) +} + +func (g *GoResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) { + return g.R.LookupIP(ctx, "ip", host) +} + +// Follow walks a CNAME chain up to maxDepth hops, detecting loops. +func Follow(ctx context.Context, r Resolver, host string, maxDepth int) ([]string, error) { + if maxDepth < 1 { + maxDepth = 1 + } + chain := []string{host} + cur := host + for i := 0; i < maxDepth; i++ { + next, err := r.LookupCNAME(ctx, cur) + if err != nil { + return chain, err + } + next = strings.TrimSuffix(next, ".") + if next == "" || next == cur { + break + } + for _, seen := range chain { + if seen == next { + return chain, ErrCNAMELoop + } + } + chain = append(chain, next) + cur = next + } + return chain, nil +} + +// Enricher resolves hostnames with caching and bounded CNAME depth. It is safe +// for concurrent use. +type Enricher struct { + Resolver Resolver + Cache *TTLCache[Result] + PositiveTTL time.Duration + NegativeTTL time.Duration + Timeout time.Duration + MaxDepth int + Metrics *metrics.Metrics +} + +// Resolve returns the result for host. ok is false when the lookup failed; +// failures (including "no such host") are negatively cached for NegativeTTL. +func (e *Enricher) Resolve(ctx context.Context, host string) (Result, bool) { + if e.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, e.Timeout) + defer cancel() + } + if e.Cache != nil { + if res, ok := e.Cache.Get(host); ok { + if e.Metrics != nil { + e.Metrics.DNSCacheHits.Add(1) + } + return res, len(res.Chain) > 0 || len(res.IPs) > 0 + } + } + if e.Metrics != nil { + e.Metrics.DNSQueries.Add(1) + } + + chain, cnameErr := Follow(ctx, e.Resolver, host, e.MaxDepth) + if cnameErr != nil && len(chain) == 1 { + // A definitive NXDOMAIN has no A/AAAA to fetch either, so skip the + // second lookup. NXDOMAIN is a valid answer, not an infrastructure error. + if e.Metrics != nil && !definitive(cnameErr) { + e.Metrics.DNSErrors.Add(1) + } + if e.Cache != nil { + e.Cache.Set(host, Result{}, e.NegativeTTL) + } + return Result{}, false + } + res := Result{Chain: chain} + if ips, err := e.Resolver.LookupIP(ctx, host); err == nil { + res.IPs = ips + } + if e.Cache != nil { + e.Cache.Set(host, res, e.PositiveTTL) + } + return res, true +} diff --git a/internal/dns/resolver_test.go b/internal/dns/resolver_test.go new file mode 100644 index 0000000..592d9d6 --- /dev/null +++ b/internal/dns/resolver_test.go @@ -0,0 +1,142 @@ +package dns + +import ( + "context" + "errors" + "net" + "testing" + "time" +) + +type fakeResolver struct { + cnames map[string]string + ips map[string][]net.IP + err error +} + +func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) { + if f.err != nil { + return "", f.err + } + if c, ok := f.cnames[host]; ok { + return c, nil + } + return host, nil +} + +func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) { + if f.err != nil { + return nil, f.err + } + return f.ips[host], nil +} + +func TestFollowChain(t *testing.T) { + r := &fakeResolver{cnames: map[string]string{ + "a.example.com": "b.example.com", + "b.example.com": "c.s3.amazonaws.com", + "c.s3.amazonaws.com": "c.s3.amazonaws.com", + }} + chain, err := Follow(context.Background(), r, "a.example.com", 8) + if err != nil { + t.Fatal(err) + } + want := []string{"a.example.com", "b.example.com", "c.s3.amazonaws.com"} + if len(chain) != len(want) { + t.Fatalf("chain = %v, want %v", chain, want) + } + for i := range want { + if chain[i] != want[i] { + t.Fatalf("chain = %v, want %v", chain, want) + } + } +} + +func TestFollowLoop(t *testing.T) { + r := &fakeResolver{cnames: map[string]string{ + "a.example.com": "b.example.com", + "b.example.com": "a.example.com", + }} + chain, err := Follow(context.Background(), r, "a.example.com", 8) + if !errors.Is(err, ErrCNAMELoop) { + t.Fatalf("err = %v, want ErrCNAMELoop", err) + } + if len(chain) < 2 { + t.Fatalf("chain = %v, want at least 2 entries", chain) + } +} + +func TestFollowDepth(t *testing.T) { + r := &fakeResolver{cnames: map[string]string{ + "h1.example.com": "h2.example.com", + "h2.example.com": "h3.example.com", + "h3.example.com": "h4.example.com", + }} + chain, err := Follow(context.Background(), r, "h1.example.com", 2) + if err != nil { + t.Fatal(err) + } + if len(chain) != 3 { + t.Fatalf("chain = %v, want 3 entries (host + 2 hops)", chain) + } +} + +func TestEnricherCaches(t *testing.T) { + r := &fakeResolver{ + cnames: map[string]string{"a.example.com": "b.s3.amazonaws.com"}, + ips: map[string][]net.IP{"a.example.com": {net.ParseIP("1.2.3.4")}}, + } + e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8} + if _, ok := e.Resolve(context.Background(), "a.example.com"); !ok { + t.Fatal("first resolve failed") + } + r.cnames = nil // a cached lookup must not hit the resolver + res, ok := e.Resolve(context.Background(), "a.example.com") + if !ok || len(res.IPs) != 1 { + t.Fatalf("cached resolve = %+v ok=%v", res, ok) + } +} + +func TestEnricherNegativeCache(t *testing.T) { + r := &fakeResolver{err: &net.DNSError{Err: "no such host", IsNotFound: true}} + e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8} + if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok { + t.Fatal("expected failure") + } + if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok { + t.Fatal("expected cached failure") + } +} + +type ctxResolver struct{} + +func (ctxResolver) LookupCNAME(ctx context.Context, host string) (string, error) { + return "", ctx.Err() +} +func (ctxResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) { + return nil, ctx.Err() +} + +func TestResolveContextCancellation(t *testing.T) { + e := &Enricher{Resolver: ctxResolver{}, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok := e.Resolve(ctx, "example.com"); ok { + t.Fatal("expected cancellation to fail resolution") + } +} + +func TestCacheBoundAndExpiry(t *testing.T) { + c := NewTTLCache[int](8, 2) + for i := 0; i < 100; i++ { + c.Set(string(rune('a'+i%26))+string(rune('0'+i%10)), i, time.Minute) + } + if c.Len() > 8 { + t.Fatalf("cache len = %d, want <= 8", c.Len()) + } + c.Set("k", 1, time.Millisecond) + time.Sleep(5 * time.Millisecond) + if _, ok := c.Get("k"); ok { + t.Fatal("expected expired entry to be gone") + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..2b49ebd --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,91 @@ +// Package metrics exposes lightweight Prometheus text-format metrics without +// pulling in a metrics dependency. Counters are plain atomics; gauges are +// callback functions so they always report current values. +package metrics + +import ( + "fmt" + "net/http" + "runtime" + "sync/atomic" +) + +type gauge struct { + name string + help string + extra string + fn func() float64 +} + +type Metrics struct { + CTCertificates atomic.Uint64 + CTIdentifiers atomic.Uint64 + + DirectMatches atomic.Uint64 + CNAMEMatches atomic.Uint64 + + DNSQueries atomic.Uint64 + DNSCacheHits atomic.Uint64 + DNSErrors atomic.Uint64 + + Candidates atomic.Uint64 + + DBRows atomic.Uint64 + DBErrors atomic.Uint64 + + QueueDrops atomic.Uint64 + QueueFull atomic.Uint64 + MalformedCT atomic.Uint64 + CTErrors atomic.Uint64 + ProbeAttempts atomic.Uint64 + ProbePublic atomic.Uint64 + + gauges []gauge +} + +func New() *Metrics { + m := &Metrics{} + m.gauges = []gauge{ + {"go_goroutines", "Number of goroutines", "", func() float64 { return float64(runtime.NumGoroutine()) }}, + {"process_system_bytes", "Bytes of memory obtained from the OS", "", func() float64 { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + return float64(ms.Sys) + }}, + } + return m +} + +// SetQueueDepth registers a gauge reporting the live depth of a named queue. +func (m *Metrics) SetQueueDepth(name string, fn func() float64) { + m.gauges = append(m.gauges, gauge{"queue_depth", "Current depth of an internal queue", `{queue="` + name + `"}`, fn}) +} + +// Handler returns an http.Handler serving the Prometheus text exposition format. +func (m *Metrics) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + counter := func(name, help string, v uint64) { + fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v) + } + counter("ct_certificates_total", "Certificates processed from CT logs", m.CTCertificates.Load()) + counter("ct_identifiers_total", "DNS identifiers parsed from CT certificates", m.CTIdentifiers.Load()) + counter("classifier_direct_matches_total", "Identifiers matching a provider directly", m.DirectMatches.Load()) + counter("classifier_cname_matches_total", "CNAME targets matching a provider", m.CNAMEMatches.Load()) + counter("dns_queries_total", "DNS queries performed", m.DNSQueries.Load()) + counter("dns_cache_hits_total", "DNS lookups served from cache", m.DNSCacheHits.Load()) + counter("dns_errors_total", "DNS lookup errors", m.DNSErrors.Load()) + counter("candidates_total", "Provider candidates produced", m.Candidates.Load()) + counter("db_rows_total", "Rows written to PostgreSQL", m.DBRows.Load()) + counter("db_errors_total", "PostgreSQL write errors", m.DBErrors.Load()) + counter("queue_drops_total", "Items dropped because a queue was full", m.QueueDrops.Load()) + counter("queue_full_total", "Times a queue was observed full", m.QueueFull.Load()) + counter("ct_malformed_entries_total", "Malformed CT log entries", m.MalformedCT.Load()) + counter("ct_errors_total", "Non-fatal CT errors", m.CTErrors.Load()) + counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load()) + counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load()) + for _, g := range m.gauges { + fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn()) + } + }) +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..880a1a2 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,219 @@ +// Package model holds the compact enums and hot-path structs shared by the +// classifier, pipeline and store. +package model + +// Provider identifies a cloud/database/storage provider. +type Provider uint8 + +const ( + ProviderUnknown Provider = iota + ProviderAWS + ProviderAzure + ProviderGCP + ProviderCloudflare + ProviderDigitalOcean + ProviderSupabase + ProviderOracle +) + +func (p Provider) String() string { + switch p { + case ProviderAWS: + return "aws" + case ProviderAzure: + return "azure" + case ProviderGCP: + return "gcp" + case ProviderCloudflare: + return "cloudflare" + case ProviderDigitalOcean: + return "digitalocean" + case ProviderSupabase: + return "supabase" + case ProviderOracle: + return "oracle" + default: + return "unknown" + } +} + +// Service identifies the provider service the resource belongs to. +type Service uint8 + +const ( + ServiceUnknown Service = iota + + ServiceObjectStorage + ServiceS3 + ServiceBlobStorage + ServiceDatabase + ServicePostgres + ServiceMySQL + ServiceRDS + ServiceDocumentDB + ServiceRedshift + ServiceOpenSearch + ServiceNeptune + ServiceRedis + ServiceMemoryDB + ServiceCosmosDB + ServiceFirebaseDB + ServiceElastiCache + ServiceSearch + ServiceQueue + ServiceFileStorage + ServiceTableStorage + ServiceWebApp +) + +func (s Service) String() string { + switch s { + case ServiceObjectStorage: + return "object_storage" + case ServiceS3: + return "s3" + case ServiceBlobStorage: + return "blob_storage" + case ServiceDatabase: + return "database" + case ServicePostgres: + return "postgres" + case ServiceMySQL: + return "mysql" + case ServiceRDS: + return "rds" + case ServiceDocumentDB: + return "documentdb" + case ServiceRedshift: + return "redshift" + case ServiceOpenSearch: + return "opensearch" + case ServiceNeptune: + return "neptune" + case ServiceRedis: + return "redis" + case ServiceMemoryDB: + return "memorydb" + case ServiceCosmosDB: + return "cosmosdb" + case ServiceFirebaseDB: + return "firebase_db" + case ServiceElastiCache: + return "elasticache" + case ServiceSearch: + return "search" + case ServiceQueue: + return "queue" + case ServiceFileStorage: + return "file_storage" + case ServiceTableStorage: + return "table_storage" + case ServiceWebApp: + return "web_app" + default: + return "unknown" + } +} + +// Source is where a candidate was discovered. +type Source uint8 + +const ( + SourceUnknown Source = iota + SourceDirectCT + SourceCNAME + SourceDNSAlias +) + +func (s Source) String() string { + switch s { + case SourceDirectCT: + return "direct_ct" + case SourceCNAME: + return "cname" + case SourceDNSAlias: + return "dns_alias" + default: + return "unknown" + } +} + +// PublicSemantic captures when the hostname format itself carries public +// accessibility meaning. It never asserts that data is actually exposed. +type PublicSemantic uint8 + +const ( + PublicSemanticNone PublicSemantic = iota + PublicSemanticPublicEndpoint + PublicSemanticVPCEndpoint + PublicSemanticPublicURL +) + +func (p PublicSemantic) String() string { + switch p { + case PublicSemanticPublicEndpoint: + return "public_endpoint" + case PublicSemanticVPCEndpoint: + return "vpc_endpoint" + case PublicSemanticPublicURL: + return "public_url" + default: + return "none" + } +} + +// ExposureState is the outcome of an optional safe reachability check. +type ExposureState uint8 + +const ( + ExposureUnknown ExposureState = iota + ExposurePublicDNS + ExposureReachable + ExposureAuthRequired + ExposurePublicRead + ExposurePrivate + ExposureNotFound +) + +func (e ExposureState) String() string { + switch e { + case ExposurePublicDNS: + return "public_dns" + case ExposureReachable: + return "reachable" + case ExposureAuthRequired: + return "auth_required" + case ExposurePublicRead: + return "public_read" + case ExposurePrivate: + return "private" + case ExposureNotFound: + return "not_found" + default: + return "unknown" + } +} + +// Match is the compact result of classifying a single hostname. The string +// fields are substrings of the input hostname (zero-copy). +type Match struct { + Provider Provider + Service Service + PublicSemantic PublicSemantic + ResourceSpecific bool + Region string + Resource string +} + +// Candidate is a provider resource discovered directly or through DNS. +type Candidate struct { + Hostname string // hostname that matched (provider hostname) + Original string // original CT hostname (equal to Hostname for direct matches) + Region string + Resource string + Provider Provider + Service Service + Source Source + PublicSemantic PublicSemantic + ResourceSpecific bool +} diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go new file mode 100644 index 0000000..bb61a39 --- /dev/null +++ b/internal/pipeline/pipeline.go @@ -0,0 +1,345 @@ +// Package pipeline wires the in-process CertSpotter identifier callback to the +// classifier, DNS enrichment and the batched PostgreSQL writer using bounded +// queues throughout. +package pipeline + +import ( + "context" + "net" + "sync" + "time" + + "software.sslmate.com/src/certspotter" + + "tea.karlcom.de/karl/bucketeer/internal/classify" + "tea.karlcom.de/karl/bucketeer/internal/dns" + "tea.karlcom.de/karl/bucketeer/internal/metrics" + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +// Finding is a fully enriched candidate ready to be persisted. +type Finding struct { + model.Candidate + CNAMEChain []string + IPv4 []string + IPv6 []string +} + +// WriteFunc persists a batch of findings. +type WriteFunc func(ctx context.Context, batch []Finding) error + +type Config struct { + DirectQueueSize int + DNSQueueSize int + DBQueueSize int + DirectWorkers int + DNSWorkers int + BatchSize int + FlushInterval time.Duration + DedupTTL time.Duration + DedupSize int + + Enricher *dns.Enricher + Write WriteFunc + Metrics *metrics.Metrics +} + +// Pipeline is the certificate-to-database data flow. It is safe to call +// HandleIdentifiers from the CertSpotter monitor goroutines concurrently. +type Pipeline struct { + cfg Config + + directQ chan model.Candidate + dnsQ chan string + dbQ chan Finding + + dedup *dns.TTLCache[bool] // hostnames already queued for DNS + seen *dns.TTLCache[bool] // provider+service+provider_hostname already written + producer sync.WaitGroup + workers sync.WaitGroup +} + +func New(cfg Config) *Pipeline { + if cfg.DirectQueueSize <= 0 { + cfg.DirectQueueSize = 8192 + } + if cfg.DNSQueueSize <= 0 { + cfg.DNSQueueSize = 8192 + } + if cfg.DBQueueSize <= 0 { + cfg.DBQueueSize = 8192 + } + if cfg.DirectWorkers <= 0 { + cfg.DirectWorkers = 2 + } + if cfg.DNSWorkers <= 0 { + cfg.DNSWorkers = 16 + } + if cfg.BatchSize <= 0 { + cfg.BatchSize = 500 + } + if cfg.FlushInterval <= 0 { + cfg.FlushInterval = 2 * time.Second + } + if cfg.DedupTTL <= 0 { + cfg.DedupTTL = time.Hour + } + if cfg.DedupSize <= 0 { + cfg.DedupSize = 1 << 20 + } + p := &Pipeline{ + cfg: cfg, + directQ: make(chan model.Candidate, cfg.DirectQueueSize), + dnsQ: make(chan string, cfg.DNSQueueSize), + dbQ: make(chan Finding, cfg.DBQueueSize), + dedup: dns.NewTTLCache[bool](cfg.DedupSize, 64), + seen: dns.NewTTLCache[bool](cfg.DedupSize, 64), + } + if p.cfg.Metrics != nil { + p.cfg.Metrics.SetQueueDepth("direct", func() float64 { return float64(len(p.directQ)) }) + p.cfg.Metrics.SetQueueDepth("dns", func() float64 { return float64(len(p.dnsQ)) }) + p.cfg.Metrics.SetQueueDepth("db", func() float64 { return float64(len(p.dbQ)) }) + } + return p +} + +// Start launches the worker pools. Close must be called to drain. +func (p *Pipeline) Start(ctx context.Context) { + for i := 0; i < p.cfg.DirectWorkers; i++ { + p.producer.Add(1) + go p.directWorker(ctx) + } + for i := 0; i < p.cfg.DNSWorkers; i++ { + p.producer.Add(1) + go p.dnsWorker(ctx) + } + p.workers.Add(1) + go p.dbWorker(ctx) +} + +// Close stops intake and drains all queues. It must only be called after the +// CertSpotter monitor has stopped calling HandleIdentifiers. +func (p *Pipeline) Close() { + close(p.directQ) + close(p.dnsQ) + p.producer.Wait() + close(p.dbQ) + p.workers.Wait() +} + +// HandleIdentifiers implements certspotter.IdentifierSink. It is deliberately +// allocation-light and never blocks: queues are bounded and overflow is dropped. +func (p *Pipeline) HandleIdentifiers(ctx context.Context, ids *certspotter.Identifiers) { + if p.cfg.Metrics != nil { + p.cfg.Metrics.CTCertificates.Add(1) + p.cfg.Metrics.CTIdentifiers.Add(uint64(len(ids.DNSNames))) + } + for _, host := range ids.DNSNames { + p.HandleHost(host) + } +} + +// HandleHost classifies a single hostname and enqueues it for direct handling +// or, when it does not match, for DNS/CNAME enrichment. +func (p *Pipeline) HandleHost(host string) { + if m, ok := classify.Classify(host); ok { + if !m.ResourceSpecific { + // Generic service endpoints (bucket name in the path) are not + // customer resources; do not create findings for them. + return + } + if p.cfg.Metrics != nil { + p.cfg.Metrics.DirectMatches.Add(1) + p.cfg.Metrics.Candidates.Add(1) + } + if !p.markSeen(host) { + return + } + cand := candidateFrom(host, host, m, model.SourceDirectCT) + select { + case p.directQ <- cand: + default: + p.drop() + } + return + } + if !classify.ShouldEnrich(host) { + return + } + if _, dup := p.dedup.Get(host); dup { + return + } + p.dedup.Set(host, true, p.cfg.DedupTTL) + select { + case p.dnsQ <- host: + default: + p.drop() + } +} + +func (p *Pipeline) drop() { + if p.cfg.Metrics != nil { + p.cfg.Metrics.QueueFull.Add(1) + p.cfg.Metrics.QueueDrops.Add(1) + } +} + +func candidateFrom(host, original string, m model.Match, src model.Source) model.Candidate { + return model.Candidate{ + Hostname: host, + Original: original, + Region: m.Region, + Resource: m.Resource, + Provider: m.Provider, + Service: m.Service, + Source: src, + PublicSemantic: m.PublicSemantic, + ResourceSpecific: m.ResourceSpecific, + } +} + +func appendIPs(f *Finding, ips []net.IP) { + for _, ip := range ips { + if v4 := ip.To4(); v4 != nil { + f.IPv4 = append(f.IPv4, v4.String()) + } else { + f.IPv6 = append(f.IPv6, ip.String()) + } + } +} + +func (p *Pipeline) enrich(ctx context.Context, c model.Candidate) Finding { + f := Finding{Candidate: c} + if p.cfg.Enricher == nil { + return f + } + res, ok := p.cfg.Enricher.Resolve(ctx, c.Hostname) + if !ok { + return f + } + f.CNAMEChain = res.Chain + appendIPs(&f, res.IPs) + return f +} + +func (p *Pipeline) directWorker(ctx context.Context) { + defer p.producer.Done() + for { + select { + case <-ctx.Done(): + return + case c, ok := <-p.directQ: + if !ok { + return + } + p.pushDB(ctx, p.enrich(ctx, c)) + } + } +} + +func (p *Pipeline) dnsWorker(ctx context.Context) { + defer p.producer.Done() + for { + select { + case <-ctx.Done(): + return + case host, ok := <-p.dnsQ: + if !ok { + return + } + if len(p.dbQ) >= cap(p.dbQ)/2 { + // Load shedding: optional CNAME enrichment is dropped before + // direct matches when the database writer is backed up. + p.drop() + continue + } + p.enrichHost(ctx, host) + } + } +} + +func (p *Pipeline) enrichHost(ctx context.Context, host string) { + res, ok := p.cfg.Enricher.Resolve(ctx, host) + if !ok { + return + } + for _, target := range res.Chain[1:] { + m, matched := classify.Classify(target) + if !matched || !m.ResourceSpecific { + continue + } + if p.cfg.Metrics != nil { + p.cfg.Metrics.CNAMEMatches.Add(1) + p.cfg.Metrics.Candidates.Add(1) + } + if !p.markSeen(target) { + continue + } + f := Finding{ + Candidate: candidateFrom(target, host, m, model.SourceCNAME), + CNAMEChain: res.Chain, + } + appendIPs(&f, res.IPs) + p.pushDB(ctx, f) + } +} + +// markSeen dedups by hostname alone: a given hostname always classifies to the +// same provider/service, so no key concatenation (and no allocation) is needed. +func (p *Pipeline) markSeen(host string) bool { + if _, dup := p.seen.Get(host); dup { + return false + } + p.seen.Set(host, true, p.cfg.DedupTTL) + return true +} + +func (p *Pipeline) pushDB(ctx context.Context, f Finding) { + select { + case p.dbQ <- f: + default: + p.drop() + } +} + +func (p *Pipeline) dbWorker(ctx context.Context) { + defer p.workers.Done() + batch := make([]Finding, 0, p.cfg.BatchSize) + ticker := time.NewTicker(p.cfg.FlushInterval) + defer ticker.Stop() + + flush := func() { + if len(batch) == 0 { + return + } + if p.cfg.Write != nil { + if err := p.cfg.Write(ctx, batch); err != nil { + if p.cfg.Metrics != nil { + p.cfg.Metrics.DBErrors.Add(1) + } + } else if p.cfg.Metrics != nil { + p.cfg.Metrics.DBRows.Add(uint64(len(batch))) + } + } + batch = batch[:0] + } + + for { + select { + case <-ctx.Done(): + flush() + return + case f, ok := <-p.dbQ: + if !ok { + flush() + return + } + batch = append(batch, f) + if len(batch) >= p.cfg.BatchSize { + flush() + } + case <-ticker.C: + flush() + } + } +} diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go new file mode 100644 index 0000000..af6ab5a --- /dev/null +++ b/internal/pipeline/pipeline_test.go @@ -0,0 +1,236 @@ +package pipeline + +import ( + "context" + "net" + "strconv" + "sync" + "testing" + "time" + + "software.sslmate.com/src/certspotter" + + "tea.karlcom.de/karl/bucketeer/internal/dns" + "tea.karlcom.de/karl/bucketeer/internal/metrics" + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +func makeHosts(prefix, suffix string) []string { + hosts := make([]string, 256) + for i := range hosts { + hosts[i] = prefix + strconv.Itoa(i) + suffix + } + return hosts +} + +// TestHandleHostNoAllocs audits the sink hot path: after warmup, classifying and +// enqueueing a hostname must not allocate. +func TestHandleHostNoAllocs(t *testing.T) { + p := New(Config{DNSQueueSize: 1 << 20, DedupSize: 1 << 20}) + nonMatch := makeHosts("api", ".example.com") + direct := makeHosts("bucket", ".s3.eu-central-1.amazonaws.com") + for _, h := range nonMatch { + p.HandleHost(h) + } + for _, h := range direct { + p.HandleHost(h) + } + allocs := testing.AllocsPerRun(2000, func() { + for i := 0; i < 64; i++ { + p.HandleHost(nonMatch[i]) + p.HandleHost(direct[i]) + } + }) + if allocs != 0 { + t.Fatalf("HandleHost allocates %.3f allocs/op, want 0", allocs) + } +} + +func BenchmarkHandleHostNonMatch(b *testing.B) { + p := New(Config{DNSQueueSize: 1 << 20, DedupSize: 1 << 20}) + hosts := makeHosts("api", ".example.com") + for _, h := range hosts { + p.HandleHost(h) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + p.HandleHost(hosts[i&255]) + } +} + +func BenchmarkHandleHostDirectMatch(b *testing.B) { + p := New(Config{DirectQueueSize: 1 << 20, DedupSize: 1 << 20}) + hosts := makeHosts("bucket", ".s3.eu-central-1.amazonaws.com") + for _, h := range hosts { + p.HandleHost(h) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + p.HandleHost(hosts[i&255]) + } +} + +type fakeResolver struct { + cnames map[string]string + ips map[string][]net.IP +} + +func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) { + if c, ok := f.cnames[host]; ok { + return c, nil + } + return host, nil +} + +func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) { + return f.ips[host], nil +} + +func newEnricher(r dns.Resolver) *dns.Enricher { + return &dns.Enricher{ + Resolver: r, + Cache: dns.NewTTLCache[dns.Result](1024, 8), + PositiveTTL: time.Minute, + NegativeTTL: time.Minute, + MaxDepth: 8, + } +} + +type collector struct { + mu sync.Mutex + items []Finding +} + +func (c *collector) write(ctx context.Context, batch []Finding) error { + c.mu.Lock() + defer c.mu.Unlock() + c.items = append(c.items, batch...) + return nil +} + +func (c *collector) all() []Finding { + c.mu.Lock() + defer c.mu.Unlock() + return append([]Finding(nil), c.items...) +} + +func TestDirectMatch(t *testing.T) { + col := &collector{} + m := metrics.New() + p := New(Config{ + Enricher: newEnricher(&fakeResolver{}), + Write: col.write, + Metrics: m, + BatchSize: 10, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{"bucket.s3.eu-central-1.amazonaws.com"}}) + time.Sleep(50 * time.Millisecond) + p.Close() + + items := col.all() + if len(items) != 1 { + t.Fatalf("got %d findings, want 1 (%+v)", len(items), items) + } + f := items[0] + if f.Provider != model.ProviderAWS || f.Service != model.ServiceS3 || f.Source != model.SourceDirectCT { + t.Fatalf("unexpected finding %+v", f) + } + if m.DirectMatches.Load() != 1 { + t.Fatalf("direct matches = %d, want 1", m.DirectMatches.Load()) + } +} + +func TestCNAMEMatch(t *testing.T) { + col := &collector{} + r := &fakeResolver{ + cnames: map[string]string{"files.example.com": "files.s3.amazonaws.com"}, + ips: map[string][]net.IP{"files.example.com": {net.ParseIP("1.2.3.4")}}, + } + m := metrics.New() + p := New(Config{Enricher: newEnricher(r), Write: col.write, Metrics: m, BatchSize: 10}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{"files.example.com"}}) + time.Sleep(50 * time.Millisecond) + p.Close() + + items := col.all() + if len(items) != 1 { + t.Fatalf("got %d findings, want 1", len(items)) + } + f := items[0] + if f.Source != model.SourceCNAME || f.Hostname != "files.s3.amazonaws.com" || f.Original != "files.example.com" { + t.Fatalf("unexpected finding %+v", f) + } + if len(f.IPv4) != 1 || f.IPv4[0] != "1.2.3.4" { + t.Fatalf("unexpected IPs %+v", f.IPv4) + } + if m.CNAMEMatches.Load() != 1 { + t.Fatalf("cname matches = %d, want 1", m.CNAMEMatches.Load()) + } +} + +func TestCNAMEMatchIPv6(t *testing.T) { + col := &collector{} + r := &fakeResolver{ + cnames: map[string]string{"db.example.com": "mydb.cluster-abc.us-east-1.rds.amazonaws.com"}, + ips: map[string][]net.IP{"db.example.com": {net.ParseIP("2606:4700::1111")}}, + } + p := New(Config{Enricher: newEnricher(r), Write: col.write, BatchSize: 10}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + p.HandleHost("db.example.com") + time.Sleep(50 * time.Millisecond) + p.Close() + items := col.all() + if len(items) != 1 || len(items[0].IPv6) != 1 || items[0].IPv6[0] != "2606:4700::1111" { + t.Fatalf("unexpected IPv6 finding: %+v", items) + } +} + +func TestQueueSaturationDrops(t *testing.T) { + m := metrics.New() + p := New(Config{DirectQueueSize: 1, DNSQueueSize: 1, Metrics: m}) + // Intentionally do not Start: the queue stays full. + p.HandleHost("bucket.s3.amazonaws.com") + p.HandleHost("other.s3.amazonaws.com") + if m.QueueDrops.Load() == 0 { + t.Fatal("expected a drop when the direct queue is full") + } + if m.Candidates.Load() != 2 { + t.Fatalf("candidates = %d, want 2", m.Candidates.Load()) + } +} + +func TestDNSDedup(t *testing.T) { + m := metrics.New() + p := New(Config{DNSQueueSize: 8, Metrics: m}) + p.HandleHost("files.example.com") + p.HandleHost("files.example.com") + if got := len(p.dnsQ); got != 1 { + t.Fatalf("dns queue len = %d, want 1", got) + } +} + +func TestGenericEndpointNotStored(t *testing.T) { + col := &collector{} + p := New(Config{Write: col.write, BatchSize: 10}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + p.HandleIdentifiers(ctx, &certspotter.Identifiers{DNSNames: []string{ + "objectstorage.us-ashburn-1.oraclecloud.com", + }}) + time.Sleep(30 * time.Millisecond) + p.Close() + if items := col.all(); len(items) != 0 { + t.Fatalf("generic endpoint produced findings: %+v", items) + } +} diff --git a/internal/probe/probe.go b/internal/probe/probe.go new file mode 100644 index 0000000..82c6d9e --- /dev/null +++ b/internal/probe/probe.go @@ -0,0 +1,142 @@ +// Package probe performs optional, minimal, non-destructive reachability checks. +// It never authenticates, downloads objects, mutates anything, or runs database +// queries; it only observes the response to a single HTTPS GET /. +package probe + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "sync" + "time" + + "tea.karlcom.de/karl/bucketeer/internal/metrics" + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +// UpdateFunc records the observed exposure state. +type UpdateFunc func(ctx context.Context, c model.Candidate, state model.ExposureState) + +type Config struct { + Workers int + Timeout time.Duration + Queue int + Metrics *metrics.Metrics + Update UpdateFunc +} + +// Prober runs a fixed pool of workers over a bounded queue. +type Prober struct { + cfg Config + queue chan model.Candidate + client *http.Client + wg sync.WaitGroup +} + +func New(cfg Config) *Prober { + if cfg.Workers <= 0 { + cfg.Workers = 4 + } + if cfg.Timeout <= 0 { + cfg.Timeout = 5 * time.Second + } + if cfg.Queue <= 0 { + cfg.Queue = 1024 + } + return &Prober{ + cfg: cfg, + queue: make(chan model.Candidate, cfg.Queue), + client: &http.Client{ + Timeout: cfg.Timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // observation only, no trust decision + MaxIdleConns: cfg.Workers * 2, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 30 * time.Second, + DialContext: (&net.Dialer{ + Timeout: cfg.Timeout, + KeepAlive: 30 * time.Second, + }).DialContext, + }, + }, + } +} + +func (p *Prober) Start(ctx context.Context) { + for i := 0; i < p.cfg.Workers; i++ { + p.wg.Add(1) + go p.worker(ctx) + } +} + +// Submit enqueues a candidate without blocking; it drops when the queue is full. +func (p *Prober) Submit(c model.Candidate) { + if p.cfg.Metrics != nil { + p.cfg.Metrics.ProbeAttempts.Add(1) + } + select { + case p.queue <- c: + default: + } +} + +func (p *Prober) Wait() { p.wg.Wait() } + +func (p *Prober) worker(ctx context.Context) { + defer p.wg.Done() + for { + select { + case <-ctx.Done(): + return + case c, ok := <-p.queue: + if !ok { + return + } + state := p.probe(ctx, c) + if p.cfg.Metrics != nil && state == model.ExposurePublicRead { + p.cfg.Metrics.ProbePublic.Add(1) + } + if p.cfg.Update != nil { + p.cfg.Update(ctx, c, state) + } + } + } +} + +func (p *Prober) probe(ctx context.Context, c model.Candidate) model.ExposureState { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+c.Hostname+"/", nil) + if err != nil { + return model.ExposureUnknown + } + resp, err := p.client.Do(req) + if err != nil { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return model.ExposurePrivate + } + return model.ExposureUnknown + } + defer resp.Body.Close() + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return model.ExposurePublicRead + case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized: + return model.ExposureAuthRequired + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: + return model.ExposureNotFound + default: + return model.ExposureReachable + } +} + +// Describe is a helper for logs/tests. +func Describe(s model.ExposureState) string { + return fmt.Sprintf("exposure=%s", s) +} diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go new file mode 100644 index 0000000..69a0200 --- /dev/null +++ b/internal/probe/probe_test.go @@ -0,0 +1,45 @@ +package probe + +import ( + "crypto/tls" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "tea.karlcom.de/karl/bucketeer/internal/model" +) + +func newProber(t *testing.T, status int) (*Prober, string) { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + })) + t.Cleanup(srv.Close) + p := New(Config{Workers: 1, Timeout: time.Second}) + // httptest uses a self-signed cert; the prober already skips verification. + _ = tls.Config{} + host := strings.TrimPrefix(srv.URL, "https://") + return p, host +} + +func TestProbeClassification(t *testing.T) { + cases := []struct { + status int + want model.ExposureState + }{ + {http.StatusOK, model.ExposurePublicRead}, + {http.StatusForbidden, model.ExposureAuthRequired}, + {http.StatusUnauthorized, model.ExposureAuthRequired}, + {http.StatusNotFound, model.ExposureNotFound}, + {http.StatusInternalServerError, model.ExposureReachable}, + } + for _, c := range cases { + p, host := newProber(t, c.status) + got := p.probe(t.Context(), model.Candidate{Hostname: host}) + if got != c.want { + t.Errorf("status %d: got %s, want %s", c.status, got, c.want) + } + } +} diff --git a/internal/store/state.go b/internal/store/state.go new file mode 100644 index 0000000..3be60d0 --- /dev/null +++ b/internal/store/state.go @@ -0,0 +1,165 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/monitor" +) + +var _ monitor.StateProvider = (*Store)(nil) + +// Prepare implements monitor.StateProvider. It applies the embedded schema so +// the container needs no writable filesystem. +func (s *Store) Prepare(ctx context.Context) error { + if err := s.Migrate(ctx); err != nil { + return err + } + return nil +} + +// PrepareLog is a no-op: checkpoint rows are created lazily by StoreLogState. +func (s *Store) PrepareLog(ctx context.Context, logID monitor.LogID) error { return nil } + +// StoreLogState persists the certspotter monitor position for a log. +func (s *Store) StoreLogState(ctx context.Context, logID monitor.LogID, state *monitor.LogState) error { + data, err := json.Marshal(state) + if err != nil { + return err + } + _, err = s.pool.Exec(ctx, ` + INSERT INTO ct_checkpoints (log_id, state, updated_at) + VALUES ($1, $2, now()) + ON CONFLICT (log_id) DO UPDATE SET state = EXCLUDED.state, updated_at = now()`, + logID.Base64URLString(), data) + return err +} + +// LoadLogState returns nil when no checkpoint exists for the log. +func (s *Store) LoadLogState(ctx context.Context, logID monitor.LogID) (*monitor.LogState, error) { + var data []byte + err := s.pool.QueryRow(ctx, `SELECT state FROM ct_checkpoints WHERE log_id = $1`, logID.Base64URLString()).Scan(&data) + if err == pgx.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + state := new(monitor.LogState) + if err := json.Unmarshal(data, state); err != nil { + return nil, fmt.Errorf("corrupt checkpoint for log %s: %w", logID.Base64URLString(), err) + } + return state, nil +} + +func sthKey(sth *cttypes.SignedTreeHead) string { + return strconv.FormatUint(sth.TreeSize, 10) + "-" + + strconv.FormatUint(sth.Timestamp, 10) + "-" + + sth.RootHash.Base64String() +} + +// StoreSTH stores an unverified signed tree head. If it already exists the +// original stored_at is preserved. +func (s *Store) StoreSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) (*monitor.StoredSTH, error) { + data, err := json.Marshal(sth) + if err != nil { + return nil, err + } + var storedAt time.Time + err = s.pool.QueryRow(ctx, ` + INSERT INTO ct_sths (log_id, sth_key, tree_size, sth, stored_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (log_id, sth_key) DO UPDATE SET log_id = EXCLUDED.log_id + RETURNING stored_at`, + logID.Base64URLString(), sthKey(sth), sth.TreeSize, data).Scan(&storedAt) + if err != nil { + return nil, err + } + return &monitor.StoredSTH{SignedTreeHead: *sth, StoredAt: storedAt}, nil +} + +// LoadSTHs loads all stored STHs for a log, sorted by tree size. +func (s *Store) LoadSTHs(ctx context.Context, logID monitor.LogID) ([]*monitor.StoredSTH, error) { + rows, err := s.pool.Query(ctx, + `SELECT sth, stored_at FROM ct_sths WHERE log_id = $1 ORDER BY tree_size ASC`, + logID.Base64URLString()) + if err != nil { + return nil, err + } + defer rows.Close() + var out []*monitor.StoredSTH + for rows.Next() { + var ( + data []byte + storedAt time.Time + ) + if err := rows.Scan(&data, &storedAt); err != nil { + return nil, err + } + var sth cttypes.SignedTreeHead + if err := json.Unmarshal(data, &sth); err != nil { + return nil, err + } + out = append(out, &monitor.StoredSTH{SignedTreeHead: sth, StoredAt: storedAt}) + } + return out, rows.Err() +} + +// RemoveSTH deletes an STH once it has been verified against the Merkle tree. +func (s *Store) RemoveSTH(ctx context.Context, logID monitor.LogID, sth *cttypes.SignedTreeHead) error { + _, err := s.pool.Exec(ctx, `DELETE FROM ct_sths WHERE log_id = $1 AND sth_key = $2`, + logID.Base64URLString(), sthKey(sth)) + return err +} + +// StoreIssuer and LoadIssuer are no-ops. ctcloud's identifier sink bypasses +// certificate-chain retrieval entirely, so the issuer cache is never consulted; +// keeping it empty also satisfies the "no certificate contents in the state +// store" requirement. The Go runtime cache handles any incidental calls. +func (s *Store) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error { + return nil +} + +func (s *Store) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) { + return nil, nil +} + +// NotifyCert is never called because the identifier sink handles certificates. +func (s *Store) NotifyCert(ctx context.Context, cert *monitor.DiscoveredCert) error { return nil } + +// NotifyMalformedEntry counts unparseable log entries without logging per entry. +func (s *Store) NotifyMalformedEntry(ctx context.Context, entry *monitor.LogEntry, parseError error) error { + if s.metrics != nil { + s.metrics.MalformedCT.Add(1) + } + return nil +} + +// NotifyHealthCheckFailure is rare (once per health-check interval at most), so +// it is safe to log. +func (s *Store) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info monitor.HealthCheckFailure) error { + log.Printf("ct health check failure: %s", info.Summary()) + return nil +} + +// NotifyError counts transient CT errors. It deliberately does not log, to keep +// logging off the high-frequency error path. +func (s *Store) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error { + if s.metrics != nil { + s.metrics.CTErrors.Add(1) + } + return nil +} + +// GetErrors returns the recent CT errors. ctcloud keeps only aggregate counters. +func (s *Store) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) { + return "", nil +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..809e3e9 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,146 @@ +// Package store owns the PostgreSQL schema, the batched findings writer and the +// restart-safe CT checkpoint state provider. +package store + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "tea.karlcom.de/karl/bucketeer/internal/metrics" + "tea.karlcom.de/karl/bucketeer/internal/model" + "tea.karlcom.de/karl/bucketeer/internal/pipeline" + "tea.karlcom.de/karl/bucketeer/migrations" +) + +// Store wraps a pgx pool with the operations ctcloud needs. +type Store struct { + pool *pgxpool.Pool + metrics *metrics.Metrics +} + +// Open connects to PostgreSQL. The pool is bounded; the caller configures the +// maximum number of connections. +func Open(ctx context.Context, databaseURL string, maxConns int) (*Store, error) { + cfg, err := pgxpool.ParseConfig(databaseURL) + if err != nil { + return nil, fmt.Errorf("invalid DATABASE_URL: %w", err) + } + if maxConns > 0 { + cfg.MaxConns = int32(maxConns) + } + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MinConns = 1 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, err + } + return &Store{pool: pool}, nil +} + +func (s *Store) SetMetrics(m *metrics.Metrics) { s.metrics = m } + +func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) } + +func (s *Store) Close() { s.pool.Close() } + +func (s *Store) Pool() *pgxpool.Pool { return s.pool } + +// Migrate applies the embedded schema idempotently. +func (s *Store) Migrate(ctx context.Context) error { + for _, name := range migrations.SQL() { + sqlBytes, err := migrations.FS.ReadFile(name) + if err != nil { + return err + } + if _, err := s.pool.Exec(ctx, string(sqlBytes)); err != nil { + return fmt.Errorf("applying %s: %w", name, err) + } + } + return nil +} + +const upsertFindingSQL = ` +WITH asset AS ( + INSERT INTO cloud_assets + (provider, service, hostname, provider_hostname, resource_identifier, + region, resource_specific, public_semantic, first_seen, last_seen) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now()) + ON CONFLICT (provider, service, provider_hostname) + DO UPDATE SET + last_seen = now(), + hostname = EXCLUDED.hostname, + resource_identifier = CASE WHEN EXCLUDED.resource_identifier = '' THEN cloud_assets.resource_identifier ELSE EXCLUDED.resource_identifier END, + region = CASE WHEN EXCLUDED.region = '' THEN cloud_assets.region ELSE EXCLUDED.region END + RETURNING id +) +INSERT INTO observations + (asset_id, source_hostname, source_type, cname_chain, ipv4, ipv6) +SELECT asset.id, $9, $10, $11, $12, $13 FROM asset` + +// WriteFindings writes one batch of findings in a single round trip. Asset rows +// are upserted and an observation is appended for each finding. +func (s *Store) WriteFindings(ctx context.Context, batch []pipeline.Finding) error { + if len(batch) == 0 { + return nil + } + var lastErr error + // ponytail: three quick attempts; the pipeline keeps running and the + // bounded DB queue sheds load if PostgreSQL stays down. + for attempt := 0; attempt < 3; attempt++ { + if err := s.writeBatch(ctx, batch); err != nil { + lastErr = err + if ctx.Err() != nil { + return err + } + backoff := time.Duration(1<. +- Minor bug fixes and robustness improvements. + +## v0.24.2 (2026-06-25) +- Extract all Common Names from Subject instead of first one in each RDN. Note + that modern certificate validators, including those used by all CT-enforcing + clients, ignore the Common Name. Therefore, any certificate missed by + certspotter due to this bug would not work in CT-enforcing clients anyways. +- Support -verbose with -version to print full build info. +- Fix some possible crashes in pathological situations. + +## v0.24.1 (2026-05-23) +- Update dependencies. +- Minor bug fixes. + +## v0.24.0 (2026-01-12) +- Add certspotter-authorize command, which lets you mark legitimate certificates + as authorized so you don't get notified about them. +- Deduplicate certificates and their corresponding precertificates, so you + receive only one notification when both are logged in CT. +- When `-no_save` is used, write certificates to a temporary location so scripts + can access them via `$CERT_FILENAME`, `$JSON_FILENAME`, and `$TEXT_FILENAME`. + +## v0.23.0 (2025-12-05) +- Avoid sending queries to logs in excess of the log's rate limit, if known. + This should improve monitoring of the TrustAsia logs. + +## v0.22.0 (2025-09-24) +- Retry failed requests more aggressively so certspotter doesn't fall behind + downloading entries. +- Support `$CERTSPOTTER_CACHE_DIR` for overriding cache directory location. +- Support `$STATE_DIRECTORY`, `$CONFIGURATION_DIRECTORY`, and `$CACHE_DIRECTORY` + when running under systemd. +- Truncate long error responses from logs, to keep size of error files down. + +## v0.21.0 (2025-07-02) +- Instead of writing log errors to stderr, save the last 7 days worth in the + state directory, and include recent errors in failed health check notifications. +- Send a meaningful User-Agent string with HTTP requests. + +## v0.20.1 (2025-06-19) +- Add resilience against sendmail hanging indefinitely. +- Add resilience against hooks which fork and keep stderr open. +- Upgrade dependencies to latest versions. +- Minor improvements to error handling, code quality, and efficiency. + +## v0.20.0 (2025-06-13) +- Remove -batch_size option, which is obsolete due to new parallel download system. +- Only print log errors to stderr if -verbose is specified. +- Fix bug that could cause unverified STHs to be deleted prematurely. +- Fail health check if log has never been successfully contacted. +- Improve -verbose. +- Improve documentation. + +## v0.19.1 (2025-05-07) +- Fix panic when retrying failed log requests. +- Properly log failed log requests. + +## v0.19.0 (2025-05-07) (RETRACTED) +- Add support for static-ct-api logs, the next generation of CT logs. +- Add support for downloading entries in parallel, to avoid backlogs when + monitoring fast-growing logs. +- Use $EMAIL environment variable to determine sender of emails. +- Remove submitct command. +- Make certspotter more resilient to system crashes such as power failures. + +## v0.18.0 (2023-11-13) +- Fix bug with downloading entries that did not materialize in practice + with any of the current logs. +- Include `Message-ID` and `Date` in outbound emails. + +## v0.17.0 (2023-10-26) +- Allow sendmail path to be configured with `$SENDMAIL_PATH`. +- Minor improvements to documentation, efficiency. + +## v0.16.0 (2023-02-21) +- Write malformed certs and failed healthchecks to filesystem so scripts + can access them. +- Automatically execute scripts under `$CERTSPOTTER_CONFIG_DIR/hooks.d` + if it exists. +- Automatically email addresses listed in `$CERTSPOTTER_CONFIG_DIR/email_recipients` + if it exists. + +## v0.15.1 (2023-02-09) +- Fix some typos in help and error messages. +- Allow version to be set via linker flag, to facilitate distro package building. + +## v0.15.0 (2023-02-08) +- **Significant behavior change**: certspotter is now intended to run as + a daemon instead of a cron job. Specifically, certspotter no longer + terminates unless it receives SIGTERM or SIGINT or there is a serious error. + You should remove certspotter from your crontab and arrange to run it as a + daemon, passing either the `-email` option or `-script` option to configure + how you want to be notified about certificates. + + Reason for this change: although using cron made sense in the early days of + Certificate Transparency, certspotter now needs to run continuously to reliably + keep up with the high growth rate of contemporary CT logs, and to gracefully + handle the many transient errors that can arise when monitoring CT. + See for background. + +- `-script` is now officially supported and can be used to execute + a command when a certificate is discovered or there is an error. For details, + see the [certspotter-script(8) man page](man/certspotter-script.md). + + Note the following changes from the experimental, undocumented `-script` + option found in previous versions: + - The script is also executed when there is an error. Consult the `$EVENT` + variable to determine why the script was executed. + - The `$DNS_NAMES` and `$IP_ADDRESSES` variables have been removed because + the OS limits the size of environment variables and some certificates have + too many identifiers. To determine a certificate's identifiers, you can + read the JSON file specified by the `$JSON_FILENAME` variable, as explained + in the [certspotter-script(8) man page](man/certspotter-script.md). + - The `$CERT_TYPE` variable has been removed because it is almost always + a serious mistake (that can make you miss malicious certificates) to treat + certificates and precertificates differently. If you are currently + using this variable to skip precertificates, stop doing that because + precertificates imply the existence of a corresponding certificate that you + **might not** be separately notified about. For more details, see + . + - New variable `$WATCH_ITEM` contains the first watch list item which + matched the certificate. + +- New `-email` option can be used to send an email when a certificate is + discovered or there is an error. Your system must have a working `sendmail` + command. + +- (Behavior change) You must specify the `-stdout` option if you want discovered + certificates to be written to stdout. This only makes sense when running + certspotter from the terminal; when running as a daemon you probably want to + use `-email` or `-script` instead. + +- Once a day, certspotter will send you a notification (per `-email` or + `-script`) if any problems are preventing it from detecting all certificates. + As in previous versions of certspotter, errors are written to stderr when they + occur, but since most errors are transient, you can now ignore stderr and rely + on the daily health check to notify you about any persistent problems that + require your attention. + +- certspotter now saves `.json` and `.txt` files alongside the `.pem` files + containing parsed representations of the certificate. + +- `.pem` files no longer have `.cert` or `.precert` in the filename. + +- certspotter will save its state periodically, and before terminating due to + SIGTERM or SIGINT, meaning it can resume monitoring without having to + re-download entries it has already processed. + +- The experimental "BygoneSSL" feature has been removed due to limited utility. + +- The `-num_workers` option has been removed. + +- The `-all_time` option has been removed. You can remove the certspotter state + directory if you want to re-download all entries. + +- The minimum supported Go version is now 1.19. + +## v0.14.0 (2022-06-13) +- Switch to Go module versioning conventions. + +## v0.13 (2022-06-13) +- Reduce minimum Go version to 1.17. +- Update install instructions. + +## v0.12 (2022-06-07) +- Retry failed log requests. This should make certspotter resilient + to rate limiting by logs. +- Add `-version` flag. +- Eliminate unnecessary dependency. certspotter now depends only on + golang.org/x packages. +- Switch to Go modules. + +## v0.11 (2021-08-17) +- Add support for contacting logs via HTTP proxies; + just set the appropriate environment variable as documented at + . +- Work around RFC 6962 ambiguity related to consistency proofs + for empty trees. + +## v0.10 (2020-04-29) +- Improve speed by processing logs in parallel +- Add `-start_at_end` option to begin monitoring new logs at the end, + which significantly speeds up Cert Spotter, at the cost of missing + certificates that were added to a log before Cert Spotter starts + monitoring it +- (Behavior change) Scan logs in their entirety the first time Cert + Spotter is run, unless `-start_at_end` specified (behavior change) +- The log list is now retrieved from certspotter.org at startup instead + of being embedded in the source. This will allow Cert Spotter to react + more quickly to the frequent changes in logs. +- (Behavior change) the `-logs` option now expects a JSON file in the v2 + log list format. See + and . +- `-logs` now accepts an HTTPS URL in addition to a file path. +- (Behavior change) the `-underwater` option has been removed. If you want + its behavior, specify `https://loglist.certspotter.org/underwater.json` to + the `-logs` option. + +## v0.9 (2018-04-19) +- Add Cloudflare Nimbus logs +- Remove Google Argon 2017 log +- Remove WoSign and StartCom logs due to disqualification by Chromium + and extended downtime + +## v0.8 (2017-12-08) +- Add Symantec Sirius log +- Add DigiCert 2 log + +## v0.7 (2017-11-13) +- Add Google Argon logs +- Fix bug that caused crash on 32 bit architectures + +## v0.6 (2017-10-19) +- Add Comodo Mammoth and Comodo Sabre logs +- Minor bug fixes and improvements + +## v0.5 (2017-05-18) +- Remove PuChuangSiDa 1 log due to excessive downtime and presumptive + disqualification from Chrome +- Add Venafi Gen2 log +- Improve monitoring robustness under certain pathological behavior + by logs +- Minor documentation improvements + +## v0.4 (2017-04-03) +- Add PuChuangSiDa 1 log +- Remove Venafi log due to fork and disqualification from Chrome + +## v0.3 (2017-02-20) +- Revise `-all_time` flag (behavior change): + - If `-all_time` is specified, scan the entirety of all logs, even + existing logs + - When a new log is added, scan it in its entirety even if `-all_time` + is not specified +- Add new logs: + - Google Icarus + - Google Skydiver + - StartCom + - WoSign +- Overhaul log processing and auditing logic: + - STHs are never deleted unless they can be verified + - Multiple unverified STHs can be queued per log, laying groundwork + for STH pollination support + - New state directory layout; current state directories will be + migrated, but migration will be removed in a future version + - Persist condensed Merkle Tree state between runs, instead of + reconstructing it from consistency proof every time +- Use a lock file to prevent multiple instances of Cert Spotter from + running concurrently (which could clobber the state directory). +- Minor bug fixes and improvements + +## v0.2 (2016-08-25) +- Suppress duplicate identifiers in output. +- Fix "EOF" error when running under Go 1.7. +- Fix bug where hook script could fail silently. +- Fix compilation under Go 1.5. + +## v0.1 (2016-07-27) +- Initial release. diff --git a/third_party/certspotter/CONTRIBUTING b/third_party/certspotter/CONTRIBUTING new file mode 100644 index 0000000..6273e7d --- /dev/null +++ b/third_party/certspotter/CONTRIBUTING @@ -0,0 +1,44 @@ +All contributors to Cert Spotter must accept the Developer Certificate +of Origin (DCO): + + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + 660 York Street, Suite 102, + San Francisco, CA 94110 USA + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + +To accept the DCO, add the following line to each commit message with +your real name and email address (`git commit -s` will do this for you): + + Signed-off-by: Alex Smith diff --git a/third_party/certspotter/CTCLOUD_PATCH.md b/third_party/certspotter/CTCLOUD_PATCH.md new file mode 100644 index 0000000..a5c4059 --- /dev/null +++ b/third_party/certspotter/CTCLOUD_PATCH.md @@ -0,0 +1,53 @@ +# ctcloud patch to upstream CertSpotter + +This directory is a copy of upstream CertSpotter +(https://github.com/SSLMate/certspotter), pulled in as a local module and +wired up with a `replace` directive in the root `go.mod`. + +It is intentionally a copy (not a git submodule) so the patched tree lives in +this repository and is compiled by the Docker build with no network access to a +separate fork. To rebase on a newer upstream release: + +1. Replace this directory with the new upstream tree. +2. Re-apply the two hunks below. + +## Changes (only two hunks) + +### `monitor/config.go` + +Adds an exported `IdentifierSink` interface and a `Config.IdentifierSink` +field: + +```go +type IdentifierSink interface { + HandleIdentifiers(ctx context.Context, identifiers *certspotter.Identifiers) +} +``` + +### `monitor/process.go` + +In `processCertificate`, immediately after `certInfo.ParseIdentifiers()`, hand +the parsed identifiers to the sink and return, bypassing the watchlist, +certificate-chain retrieval and notification paths: + +```go +if config.IdentifierSink != nil { + config.IdentifierSink.HandleIdentifiers(ctx, identifiers) + return nil +} +``` + +## Why + +The sink gives ctcloud parsed DNS identifiers via a direct in-process Go call: + +```text +CT log entry -> CertSpotter parse -> IdentifierSink.HandleIdentifiers -> classifier +``` + +No stdout, no JSON, no files, no hooks, no subprocess. Setting the sink also +skips issuer-chain fetching and the watchlist machinery, which is exactly the +I/O and serialization overhead the design forbids. + +Upstream is otherwise unmodified. License and notices are preserved in +`LICENSE`. diff --git a/third_party/certspotter/LICENSE b/third_party/certspotter/LICENSE new file mode 100644 index 0000000..14e2f77 --- /dev/null +++ b/third_party/certspotter/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/third_party/certspotter/README.md b/third_party/certspotter/README.md new file mode 100644 index 0000000..a65dea6 --- /dev/null +++ b/third_party/certspotter/README.md @@ -0,0 +1,183 @@ +# Cert Spotter - Certificate Transparency Monitor - Open Source Edition + +**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that +alerts you when an SSL/TLS certificate is issued for one of your domains. +Cert Spotter is easier to use than other open source CT monitors, since it does not require +a database. It's also more robust, since it uses a special certificate parser +that ensures it won't miss certificates. + +Cert Spotter is available as a hosted service by SSLMate that also +provides monitoring for expiring and incorrectly-installed certificates, +Slack notifications, and a Web dashboard for tracking all of your +certificates. Visit to sign up. + +You can use Cert Spotter to detect: + + * Certificates issued to attackers who have compromised your DNS and + are redirecting your visitors to their malicious site. + * Certificates issued to attackers who have taken over an abandoned + sub-domain in order to serve malware under your name. + * Certificates issued to attackers who have compromised a certificate + authority and want to impersonate your site. + * Certificates issued in violation of your corporate policy + or outside of your centralized certificate procurement process. + +## Quickstart + +The following instructions require you to have [Go version 1.21 or higher](https://go.dev/dl/) installed. + +1. Install the certspotter command using the `go` command: + + ``` + go install software.sslmate.com/src/certspotter/cmd/certspotter@latest + ``` + +2. Create a watch list file `$HOME/.certspotter/watchlist` containing the DNS names you want to monitor, + one per line. To monitor an entire domain tree (including the domain itself + and all sub-domains) prefix the domain name with a dot (e.g. `.example.com`). + To monitor a single DNS name only, do not prefix the name with a dot. + +3. Place one or more email addresses in the `$HOME/.certspotter/email_recipients` + file (one per line), and/or place one or more executable scripts in the + `$HOME/.certspotter/hooks.d` directory. certspotter will email the listed + addresses (requires your system to have a working sendmail command) and + execute the provided scripts when it detects a certificate for a domain on + your watch list. + +4. Configure your system to run `certspotter` as a daemon. You may want to specify + the `-start_at_end` command line option to tell certspotter to start monitoring + new logs at the end instead of the beginning. This saves significant bandwidth, but + you won't be notified about certificates which were logged before you started + using certspotter. + +## Documentation + +* Command line options and operational details: [certspotter(8) man page](man/certspotter.md) +* The script interface: [certspotter-script(8) man page](man/certspotter-script.md) +* Authorizing known certificates: [certspotter-authorize(8) man page](man/certspotter-authorize.md) +* Monitoring certspotter with Prometheus: [certspotter-metrics(8) man page](man/certspotter-metrics.md) +* [Change Log](CHANGELOG.md) + +## Authorizing Known Certificates to Prevent False Alarms + +You can use the **certspotter-authorize** command to tell certspotter +about legitimate certificates issued by your certificate authority. +certspotter won't notify you when it discovers an authorized certificate +(or its corresponding precertificate) in Certificate Transparency logs. + +To install certspotter-authorize, run: + +``` +go install software.sslmate.com/src/certspotter/cmd/certspotter-authorize@latest +``` + +To authorize a certificate, run: + +``` +certspotter-authorize /path/to/cert.pem +``` + +For more details, see the [certspotter-authorize(8) man page](man/certspotter-authorize.md). + +## What certificates are detected by Cert Spotter? + +In the default configuration, any certificate that is logged to a +Certificate Transparency log listed as Qualified, Usable, or ReadOnly +in the Google Chrome or Apple log lists will be detected by Cert Spotter. +By default, Chrome, Safari, Firefox, and Edge only accept certificates that are +logged to at least one such log (Firefox and Edge use the Chrome log list), so any +certificate that works in these browsers will be detected by Cert Spotter. + +Cert Spotter monitors both traditional RFC6962 logs and modern static-ct-api logs. + +## Security + +Cert Spotter assumes an adversarial model in which an attacker produces +a certificate that is accepted by at least some clients but goes +undetected because of an encoding error that prevents CT monitors from +understanding it. To defend against this attack, Cert Spotter uses a +special certificate parser that keeps the certificate unparsed except +for the identifiers. If one of the identifiers matches a domain on your +watchlist, you will be notified, even if other parts of the certificate +are unparsable. + +Cert Spotter takes special precautions to ensure identifiers are parsed +correctly, and implements defenses against identifier-based attacks. +For instance, if a DNS identifier contains a null byte, Cert Spotter +interprets it as two identifiers: the complete identifier, and the +identifier formed by truncating at the first null byte. For example, a +certificate for `example.org\0.example.com` will alert the owners of both +`example.org` and `example.com`. This defends against [null prefix attacks]( +http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf). + +SSLMate continuously monitors CT logs to make sure every certificate's +identifiers can be successfully parsed, and will release updates to +Cert Spotter as necessary to fix parsing failures. + +Cert Spotter understands wildcard DNS names, and will alert +you if a wildcard certificate might match an identifier on +your watchlist. For example, a watchlist entry for `sub.example.com` would +match certificates for `*.example.com`. + +Cert Spotter is not just a log monitor, but also a log auditor which +checks that the log is obeying its append-only property. A future +release of Cert Spotter will support gossiping with other log monitors +to ensure the log is presenting a single view. + +## Testing + +Every day at around 12:05 UTC, a certificate is issued and logged to CT +for the DNS name `DAY-OF-WEEK.test.certspotter.org`. You can add one +or more of these DNS names (or `.test.certspotter.org`) to your watch +list to get confirmation that Cert Spotter is working. + +| DNS Name | Issuer | Key Type | +| -------------------------------- | ------------- | -------- | +| `monday.test.certspotter.org` | Sectigo | RSA | +| `tuesday.test.certspotter.org` | Let's Encrypt | ECDSA | +| `wednesday.test.certspotter.org` | Google | RSA | +| `thursday.test.certspotter.org` | Sectigo | ECDSA | +| `friday.test.certspotter.org` | Let's Encrypt | RSA | +| `saturday.test.certspotter.org` | Google | ECDSA | +| `sunday.test.certspotter.org` | Let's Encrypt | ECDSA | + +## Monitoring certspotter with Prometheus (Experimental) + +You can use the experimental **certspotter-metrics** command to output Prometheus metrics +about certspotter's progress monitoring each CT log, such as how many entries +have been downloaded and verified. certspotter-metrics reads the certspotter +state directory and writes metrics to stdout; run it periodically and expose +the output to Prometheus, e.g. with the node_exporter textfile collector. + +To install certspotter-metrics, run: + +``` +go install software.sslmate.com/src/certspotter/cmd/certspotter-metrics@latest +``` + +Here's a crontab entry that exposes metrics to the node_exporter textfile +collector every five minutes: + +``` +*/5 * * * * root certspotter-metrics > /var/lib/node_exporter/certspotter.prom.tmp && mv /var/lib/node_exporter/certspotter.prom.tmp /var/lib/node_exporter/certspotter.prom +``` + +Here's a Prometheus query to determine download backlog (number entries +in the log that haven't been downloaded yet): + +``` +certspotter_log_max_sth_size-certspotter_log_download_position +``` + +certspotter-metrics is experimental. Metrics may change in future +versions, and a future version of certspotter might export metrics +directly from a builtin HTTP server rather than using a separate command. Please +provide feedback at . + +For more details, see the [certspotter-metrics(8) man page](man/certspotter-metrics.md). + +## Copyright + +Copyright © 2016-2026 Opsmate, Inc. + +Licensed under the [Mozilla Public License Version 2.0](LICENSE). diff --git a/third_party/certspotter/asn1.go b/third_party/certspotter/asn1.go new file mode 100644 index 0000000..d50e495 --- /dev/null +++ b/third_party/certspotter/asn1.go @@ -0,0 +1,78 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "bytes" + "encoding/asn1" + "encoding/binary" + "errors" + "unicode/utf8" +) + +func stringFromByteSlice(chars []byte) string { + runes := make([]rune, len(chars)) + for i, ch := range chars { + runes[i] = rune(ch) + } + return string(runes) +} + +func stringFromUint16Slice(chars []uint16) string { + runes := make([]rune, len(chars)) + for i, ch := range chars { + runes[i] = rune(ch) + } + return string(runes) +} + +func stringFromUint32Slice(chars []uint32) string { + runes := make([]rune, len(chars)) + for i, ch := range chars { + runes[i] = rune(ch) + } + return string(runes) +} + +func decodeASN1String(value *asn1.RawValue) (string, error) { + if !value.IsCompound && value.Class == 0 { + if value.Tag == 12 { + // UTF8String + if !utf8.Valid(value.Bytes) { + return "", errors.New("malformed UTF8String") + } + return string(value.Bytes), nil + } else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 { + // * PrintableString - subset of ASCII + // * IA5String - ASCII + // * TeletexString - 8 bit charset; not quite ISO-8859-1, but often treated as such + // * VisibleString - subset of ASCII + + // Don't enforce character set rules. Allow any 8 bit character, since + // CAs routinely mess this up + return stringFromByteSlice(value.Bytes), nil + } else if value.Tag == 30 { + // BMPString - Unicode, encoded in big-endian format using two octets + runes := make([]uint16, len(value.Bytes)/2) + if err := binary.Read(bytes.NewReader(value.Bytes), binary.BigEndian, runes); err != nil { + return "", errors.New("Malformed BMPString: " + err.Error()) + } + return stringFromUint16Slice(runes), nil + } else if value.Tag == 28 { + // UniversalString - Unicode, encoded in big-endian format using four octets + runes := make([]uint32, len(value.Bytes)/4) + if err := binary.Read(bytes.NewReader(value.Bytes), binary.BigEndian, runes); err != nil { + return "", errors.New("Malformed UniversalString: " + err.Error()) + } + return stringFromUint32Slice(runes), nil + } + } + return "", errors.New("not a string") +} diff --git a/third_party/certspotter/asn1time.go b/third_party/certspotter/asn1time.go new file mode 100644 index 0000000..f356d14 --- /dev/null +++ b/third_party/certspotter/asn1time.go @@ -0,0 +1,257 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "encoding/asn1" + "errors" + "strconv" + "time" + "unicode" +) + +const ( + tagUTCTime = 23 + tagGeneralizedTime = 24 +) + +func isDigit(b byte) bool { + return unicode.IsDigit(rune(b)) +} + +func bytesToInt(bytes []byte) (int, error) { + return strconv.Atoi(string(bytes)) +} + +func parseUTCTime(bytes []byte) (time.Time, error) { + var err error + var year, month, day int + var hour, min, sec int + var tz *time.Location + + // YYMMDDhhmm + if len(bytes) < 10 { + return time.Time{}, errors.New("UTCTime is too short") + } + year, err = bytesToInt(bytes[0:2]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + month, err = bytesToInt(bytes[2:4]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + day, err = bytesToInt(bytes[4:6]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + hour, err = bytesToInt(bytes[6:8]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + min, err = bytesToInt(bytes[8:10]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + bytes = bytes[10:] + + // (optional) ss + if len(bytes) >= 2 && isDigit(bytes[0]) { + sec, err = bytesToInt(bytes[0:2]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + bytes = bytes[2:] + } + + // timezone (required but allow it to be omitted, since this is a common error) + if len(bytes) >= 1 { + if bytes[0] == 'Z' { + tz = time.UTC + bytes = bytes[1:] + } else if bytes[0] == '+' { + // +hhmm + if len(bytes) < 5 { + return time.Time{}, errors.New("UTCTime positive timezone offset is too short") + } + tzHour, err := bytesToInt(bytes[1:3]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + tzMin, err := bytesToInt(bytes[3:5]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + tz = time.FixedZone("", tzHour*3600+tzMin*60) + bytes = bytes[5:] + } else if bytes[0] == '-' { + // -hhmm + if len(bytes) < 5 { + return time.Time{}, errors.New("UTCTime negative timezone offset is too short") + } + tzHour, err := bytesToInt(bytes[1:3]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + tzMin, err := bytesToInt(bytes[3:5]) + if err != nil { + return time.Time{}, errors.New("UTCTime contains invalid integer: " + err.Error()) + } + + tz = time.FixedZone("", -1*(tzHour*3600+tzMin*60)) + bytes = bytes[5:] + } + } else { + tz = time.UTC + } + + if len(bytes) > 0 { + return time.Time{}, errors.New("UTCTime has trailing garbage") + } + + // https://tools.ietf.org/html/rfc5280#section-4.1.2.5.1 + if year >= 50 { + year = 1900 + year + } else { + year = 2000 + year + } + + return time.Date(year, time.Month(month), day, hour, min, sec, 0, tz), nil +} + +func parseGeneralizedTime(bytes []byte) (time.Time, error) { + var err error + var year, month, day int + var hour, min, sec, ms int + var tz *time.Location + + // YYYYMMDDHH + if len(bytes) < 10 { + return time.Time{}, errors.New("GeneralizedTime is too short") + } + year, err = bytesToInt(bytes[0:4]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + month, err = bytesToInt(bytes[4:6]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + day, err = bytesToInt(bytes[6:8]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + hour, err = bytesToInt(bytes[8:10]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + bytes = bytes[10:] + + // (optional) MM + if len(bytes) >= 2 && isDigit(bytes[0]) { + min, err = bytesToInt(bytes[0:2]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + bytes = bytes[2:] + // (optional) SS + if len(bytes) >= 2 && isDigit(bytes[0]) { + sec, err = bytesToInt(bytes[0:2]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + bytes = bytes[2:] + // (optional) .fff + if len(bytes) >= 1 && bytes[0] == '.' { + if len(bytes) < 4 { + return time.Time{}, errors.New("GeneralizedTime fractional seconds is too short") + } + ms, err = bytesToInt(bytes[1:4]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + bytes = bytes[4:] + } + } + } + + // timezone (Z or +hhmm or -hhmm or nothing) + if len(bytes) >= 1 { + if bytes[0] == 'Z' { + bytes = bytes[1:] + tz = time.UTC + } else if bytes[0] == '+' { + // +hhmm + if len(bytes) < 5 { + return time.Time{}, errors.New("GeneralizedTime positive timezone offset is too short") + } + tzHour, err := bytesToInt(bytes[1:3]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + tzMin, err := bytesToInt(bytes[3:5]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + tz = time.FixedZone("", tzHour*3600+tzMin*60) + bytes = bytes[5:] + } else if bytes[0] == '-' { + // -hhmm + if len(bytes) < 5 { + return time.Time{}, errors.New("GeneralizedTime negative timezone offset is too short") + } + tzHour, err := bytesToInt(bytes[1:3]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + tzMin, err := bytesToInt(bytes[3:5]) + if err != nil { + return time.Time{}, errors.New("GeneralizedTime contains invalid integer: " + err.Error()) + } + + tz = time.FixedZone("", -1*(tzHour*3600+tzMin*60)) + bytes = bytes[5:] + } + } else { + tz = time.UTC + } + + if len(bytes) > 0 { + return time.Time{}, errors.New("GeneralizedTime has trailing garbage") + } + + return time.Date(year, time.Month(month), day, hour, min, sec, ms*1000*1000, tz), nil +} + +func decodeASN1Time(value *asn1.RawValue) (time.Time, error) { + if !value.IsCompound && value.Class == 0 { + if value.Tag == tagUTCTime { + return parseUTCTime(value.Bytes) + } else if value.Tag == tagGeneralizedTime { + return parseGeneralizedTime(value.Bytes) + } + } + return time.Time{}, errors.New("not a time value") +} diff --git a/third_party/certspotter/asn1time_test.go b/third_party/certspotter/asn1time_test.go new file mode 100644 index 0000000..9ecbec2 --- /dev/null +++ b/third_party/certspotter/asn1time_test.go @@ -0,0 +1,121 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "testing" + "time" +) + +type timeTest struct { + in string + ok bool + out time.Time +} + +var utcTimeTests = []timeTest{ + {"9502101525Z", true, time.Date(1995, time.February, 10, 15, 25, 0, 0, time.UTC)}, + {"950210152542Z", true, time.Date(1995, time.February, 10, 15, 25, 42, 0, time.UTC)}, + {"1502101525Z", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)}, + {"150210152542Z", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)}, + {"1502101525+1000", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", 10*3600))}, + {"1502101525-1000", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", -1*(10*3600)))}, + {"1502101525+1035", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", 10*3600+35*60))}, + {"1502101525-1035", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", -1*(10*3600+35*60)))}, + {"150210152542+1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", 10*3600))}, + {"150210152542-1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", -1*(10*3600)))}, + {"150210152542+1035", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", 10*3600+35*60))}, + {"150210152542-1035", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", -1*(10*3600+35*60)))}, + {"1502101525", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)}, + {"150210152542", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)}, + {"", false, time.Time{}}, + {"123", false, time.Time{}}, + {"150210152542-10", false, time.Time{}}, + {"150210152542F", false, time.Time{}}, + {"150210152542ZF", false, time.Time{}}, +} + +func TestUTCTime(t *testing.T) { + for i, test := range utcTimeTests { + ret, err := parseUTCTime([]byte(test.in)) + if err != nil { + if test.ok { + t.Errorf("#%d: parseUTCTime(%q) failed with error %v", i, test.in, err) + } + continue + } + if !test.ok { + t.Errorf("#%d: parseUTCTime(%q) succeeded, should have failed", i, test.in) + continue + } + if !test.out.Equal(ret) { + t.Errorf("#%d: parseUTCTime(%q) = %v, want %v", i, test.in, ret, test.out) + } + } +} + +var generalizedTimeTests = []timeTest{ + {"2015021015", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.UTC)}, + {"201502101525", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)}, + {"20150210152542", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)}, + {"20150210152542.123", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.UTC)}, + {"20150210152542.12", false, time.Time{}}, + {"20150210152542.1", false, time.Time{}}, + {"20150210152542.", false, time.Time{}}, + + {"2015021015Z", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.UTC)}, + {"201502101525Z", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.UTC)}, + {"20150210152542Z", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.UTC)}, + {"20150210152542.123Z", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.UTC)}, + {"20150210152542.12Z", false, time.Time{}}, + {"20150210152542.1Z", false, time.Time{}}, + {"20150210152542.Z", false, time.Time{}}, + + {"2015021015+1000", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.FixedZone("", 10*3600))}, + {"201502101525+1000", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", 10*3600))}, + {"20150210152542+1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", 10*3600))}, + {"20150210152542.123+1000", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.FixedZone("", 10*3600))}, + {"20150210152542.12+1000", false, time.Time{}}, + {"20150210152542.1+1000", false, time.Time{}}, + {"20150210152542.+1000", false, time.Time{}}, + + {"2015021015-0835", true, time.Date(2015, time.February, 10, 15, 0, 0, 0, time.FixedZone("", -1*(8*3600+35*60)))}, + {"201502101525-0835", true, time.Date(2015, time.February, 10, 15, 25, 0, 0, time.FixedZone("", -1*(8*3600+35*60)))}, + {"20150210152542-0835", true, time.Date(2015, time.February, 10, 15, 25, 42, 0, time.FixedZone("", -1*(8*3600+35*60)))}, + {"20150210152542.123-0835", true, time.Date(2015, time.February, 10, 15, 25, 42, 123000000, time.FixedZone("", -1*(8*3600+35*60)))}, + {"20150210152542.12-0835", false, time.Time{}}, + {"20150210152542.1-0835", false, time.Time{}}, + {"20150210152542.-0835", false, time.Time{}}, + + {"", false, time.Time{}}, + {"123", false, time.Time{}}, + {"2015021015+1000Z", false, time.Time{}}, + {"2015021015x", false, time.Time{}}, + {"201502101525Zf", false, time.Time{}}, +} + +func TestGeneralizedTime(t *testing.T) { + for i, test := range generalizedTimeTests { + ret, err := parseGeneralizedTime([]byte(test.in)) + if err != nil { + if test.ok { + t.Errorf("#%d: parseGeneralizedTime(%q) failed with error %v", i, test.in, err) + } + continue + } + if !test.ok { + t.Errorf("#%d: parseGeneralizedTime(%q) succeeded, should have failed", i, test.in) + continue + } + if !test.out.Equal(ret) { + t.Errorf("#%d: parseGeneralizedTime(%q) = %v, want %v", i, test.in, ret, test.out) + } + } +} diff --git a/third_party/certspotter/canonicalize.go b/third_party/certspotter/canonicalize.go new file mode 100644 index 0000000..57f9814 --- /dev/null +++ b/third_party/certspotter/canonicalize.go @@ -0,0 +1,92 @@ +// Copyright (C) 2019 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "encoding/asn1" +) + +func canonicalizeRDNString(fromStr string) string { + from := []byte(fromStr) + to := []byte{} + inWhitespace := true + for _, ch := range from { + if ch == ' ' || ch == '\f' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\v' { + if !inWhitespace { + to = append(to, ' ') + } + inWhitespace = true + } else { + if ch >= 'A' && ch <= 'Z' { + to = append(to, ch+32) // convert to lowercase + } else { + to = append(to, ch) + } + inWhitespace = false + } + } + if inWhitespace && len(to) > 0 { + // whack off the space character that we appended + to = to[:len(to)-1] + } + return string(to) +} + +func shouldCanonicalizeASN1String(value *asn1.RawValue) bool { + if !value.IsCompound && value.Class == 0 { + return value.Tag == 12 || value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 || value.Tag == 30 || value.Tag == 28 + } + return false +} + +func canonicalizeATV(oldATV AttributeTypeAndValue) (AttributeTypeAndValue, error) { + if shouldCanonicalizeASN1String(&oldATV.Value) { + str, err := decodeASN1String(&oldATV.Value) + if err != nil { + return AttributeTypeAndValue{}, err + } + str = canonicalizeRDNString(str) + return AttributeTypeAndValue{ + Type: oldATV.Type, + Value: asn1.RawValue{ + Class: 0, + Tag: asn1.TagUTF8String, + IsCompound: false, + Bytes: []byte(str), + }, + }, nil + } else { + return oldATV, nil + } +} + +func canonicalizeRDNSet(oldSet RelativeDistinguishedNameSET) (RelativeDistinguishedNameSET, error) { + newSet := make([]AttributeTypeAndValue, len(oldSet)) + for i := range oldSet { + var err error + newSet[i], err = canonicalizeATV(oldSet[i]) + if err != nil { + return nil, err + } + } + return newSet, nil +} + +func CanonicalizeRDNSequence(oldSequence RDNSequence) (RDNSequence, error) { + newSequence := make([]RelativeDistinguishedNameSET, len(oldSequence)) + for i := range oldSequence { + var err error + newSequence[i], err = canonicalizeRDNSet(oldSequence[i]) + if err != nil { + return nil, err + } + } + return newSequence, nil +} diff --git a/third_party/certspotter/canonicalize_test.go b/third_party/certspotter/canonicalize_test.go new file mode 100644 index 0000000..134ba5d --- /dev/null +++ b/third_party/certspotter/canonicalize_test.go @@ -0,0 +1,49 @@ +// Copyright (C) 2019 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "testing" +) + +type stringCanonTest struct { + in string + out string +} + +var stringCanonTests = []stringCanonTest{ + {"", ""}, + {" ", ""}, + {" ", ""}, + {"abc", "abc"}, + {"aBc", "abc"}, + {"ab c", "ab c"}, + {"ab c", "ab c"}, + {"ab\n c", "ab c"}, + {" ab c ", "ab c"}, + {" ab c ", "ab c"}, + {" ab c", "ab c"}, + {"ab c ", "ab c"}, + {"abc ", "abc"}, + {"abc ", "abc"}, + {" abc ", "abc"}, + {" abc ", "abc"}, + {" abc", "abc"}, + {" aBc de f g\n", "abc de f g"}, +} + +func TestCanonicalizeRDNString(t *testing.T) { + for i, test := range stringCanonTests { + ret := canonicalizeRDNString(test.in) + if test.out != ret { + t.Errorf("#%d: canonicalizeRDNString(%q) = %q, want %q", i, test.in, ret, test.out) + } + } +} diff --git a/third_party/certspotter/cmd/certspotter-authorize/.gitignore b/third_party/certspotter/cmd/certspotter-authorize/.gitignore new file mode 100644 index 0000000..ce28fbd --- /dev/null +++ b/third_party/certspotter/cmd/certspotter-authorize/.gitignore @@ -0,0 +1 @@ +/certspotter-authorize diff --git a/third_party/certspotter/cmd/certspotter-authorize/main.go b/third_party/certspotter/cmd/certspotter-authorize/main.go new file mode 100644 index 0000000..f31459d --- /dev/null +++ b/third_party/certspotter/cmd/certspotter-authorize/main.go @@ -0,0 +1,202 @@ +// Copyright (C) 2026 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/pem" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime/debug" + "strconv" + "strings" + + "software.sslmate.com/src/certspotter" +) + +var programName = os.Args[0] +var Version = "unknown" +var Source = "unknown" + +func certspotterVersion() (string, string) { + if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") { + return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path + } else { + return Version, Source + } +} + +func homedir() string { + homedir, err := os.UserHomeDir() + if err != nil { + panic(fmt.Errorf("unable to determine home directory: %w", err)) + } + return homedir +} + +func startedBySupervisor() bool { + return os.Getenv("SYSTEMD_EXEC_PID") == strconv.Itoa(os.Getpid()) +} + +func defaultStateDir() string { + if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" { + return envVar + } else if envVar := os.Getenv("STATE_DIRECTORY"); envVar != "" && startedBySupervisor() { + return envVar + } else { + return filepath.Join(homedir(), ".certspotter") + } +} + +func fileExists(filename string) bool { + _, err := os.Lstat(filename) + return err == nil +} + +func readCertFile(path string) ([]byte, error) { + if path == "-" { + return io.ReadAll(os.Stdin) + } else { + return os.ReadFile(path) + } +} + +func parseCertificate(certBytes []byte) ([]byte, error) { + block, _ := pem.Decode(certBytes) + if block != nil { + if block.Type == "CERTIFICATE" { + return block.Bytes, nil + } + return nil, fmt.Errorf("PEM block type is %q, expected CERTIFICATE", block.Type) + } + return nil, fmt.Errorf("no PEM data found") +} + +func computeTBSHash(certDER []byte) ([32]byte, error) { + certInfo, err := certspotter.MakeCertInfoFromRawCert(certDER) + if err != nil { + return [32]byte{}, fmt.Errorf("error parsing certificate: %w", err) + } + precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS) + if err != nil { + return [32]byte{}, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err) + } + return sha256.Sum256(precertTBS.Raw), nil +} + +func createNotifiedMarker(stateDir string, tbsHash [32]byte) (string, error) { + tbsHex := hex.EncodeToString(tbsHash[:]) + + certsDir := filepath.Join(stateDir, "certs") + tbsDir := filepath.Join(certsDir, tbsHex[0:2]) + notifiedPath := filepath.Join(tbsDir, "."+tbsHex+".notified") + + // Check if already notified + if fileExists(notifiedPath) { + return notifiedPath, nil + } + + // Create certs directory if needed + if err := os.Mkdir(certsDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return "", fmt.Errorf("error creating certs directory: %w", err) + } + + // Create TBS-specific subdirectory if needed + if err := os.Mkdir(tbsDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return "", fmt.Errorf("error creating directory: %w", err) + } + + // Create marker file + if err := os.WriteFile(notifiedPath, nil, 0666); err != nil { + return "", fmt.Errorf("error creating marker file: %w", err) + } + + return notifiedPath, nil +} + +func main() { + version, source := certspotterVersion() + + var flags struct { + stateDir string + printhash bool + version bool + } + + flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "State directory used by certspotter") + flag.BoolVar(&flags.printhash, "printhash", false, "Instead of authorizing certificate, print its TBS hash") + flag.BoolVar(&flags.version, "version", false, "Print version and exit") + flag.Parse() + + if flags.version { + fmt.Fprintf(os.Stdout, "certspotter-authorize version %s (%s)\n", version, source) + os.Exit(0) + } + + args := flag.Args() + if len(args) == 0 { + fmt.Fprintf(os.Stderr, "Usage: %s [options] -|CERTFILE...\n", programName) + fmt.Fprintf(os.Stderr, "Purpose: suppress future certspotter notifications for a certificate and its corresponding precertificate.\n") + fmt.Fprintf(os.Stderr, "Options:\n") + flag.PrintDefaults() + os.Exit(2) + } + + if len(args) > 1 { + for _, arg := range args { + if arg == "-" { + fmt.Fprintf(os.Stderr, "%s: '-' must be the only argument when used\n", programName) + os.Exit(2) + } + } + } + + for _, certPath := range args { + certBytes, err := readCertFile(certPath) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error reading certificate: %s\n", programName, err) + os.Exit(1) + } + + certDER, err := parseCertificate(certBytes) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s: %s\n", programName, certPath, err) + os.Exit(1) + } + + tbsHash, err := computeTBSHash(certDER) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s: %s\n", programName, certPath, err) + os.Exit(1) + } + + if flags.printhash { + if certPath == "-" { + fmt.Println(hex.EncodeToString(tbsHash[:])) + } else { + fmt.Printf("%s %s\n", hex.EncodeToString(tbsHash[:]), certPath) + } + } else { + _, err = createNotifiedMarker(flags.stateDir, tbsHash) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) + os.Exit(1) + } + } + } + + os.Exit(0) +} diff --git a/third_party/certspotter/cmd/certspotter-authorize/main_test.go b/third_party/certspotter/cmd/certspotter-authorize/main_test.go new file mode 100644 index 0000000..2836616 --- /dev/null +++ b/third_party/certspotter/cmd/certspotter-authorize/main_test.go @@ -0,0 +1,190 @@ +// Copyright (C) 2026 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package main + +import ( + "bytes" + "encoding/hex" + "os" + "path/filepath" + "testing" +) + +const testCertPEM = ` +-----BEGIN CERTIFICATE----- +MIIGuzCCBSOgAwIBAgIRANubk4g/6c+TF8jITzhFX44wDQYJKoZIhvcNAQELBQAw +YDELMAkGA1UEBhMCR0IxGDAWBgNVBAoTD1NlY3RpZ28gTGltaXRlZDE3MDUGA1UE +AxMuU2VjdGlnbyBQdWJsaWMgU2VydmVyIEF1dGhlbnRpY2F0aW9uIENBIERWIFIz +NjAeFw0yNTEyMDIwMDAwMDBaFw0yNjExMjEyMzU5NTlaMBYxFDASBgNVBAMTC3Nz +bG1hdGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA235/3Y/E +4yPAHPa37C7Fgp7KPVjjuTB5vKV9nYIJzfp7NgvDBlf7k5bZFCSsSIj2txhL0hzX +Bwvmy7u7CYR7CApr2Rx2UPOl7Gmlt/DmtfyKac8Iunn2ozuGZDtxq19Go4NL9jl9 +e9O3H/lcL/ZFqzbUNlKIOfkOYkOxM3qpQXHTXuhkeI2MJO/S4wX8y8/8uhArWQ9e +h/YrtJlO9fla60kLUlQF7mtJTc+0oB3+N4eF5t2a8Pav00T6lVvH8hMhbY0nZ/tB +CD6/I6yelh8cP094VRJEGWs+zcEuXpz4FsZggkhF/l+AhQ+DfgxZhno4M60kBKC8 +Un1BTGX5TjfjJQIDAQABo4IDODCCAzQwHwYDVR0jBBgwFoAUaMASFhgOr872h6Yy +V6NGUV3LBycwHQYDVR0OBBYEFKg2kl8xIzdSxjOEAzlNpdpigAazMA4GA1UdDwEB +/wQEAwIFoDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMEkGA1Ud +IARCMEAwNAYLKwYBBAGyMQECAgcwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9zZWN0 +aWdvLmNvbS9DUFMwCAYGZ4EMAQIBMIGEBggrBgEFBQcBAQR4MHYwTwYIKwYBBQUH +MAKGQ2h0dHA6Ly9jcnQuc2VjdGlnby5jb20vU2VjdGlnb1B1YmxpY1NlcnZlckF1 +dGhlbnRpY2F0aW9uQ0FEVlIzNi5jcnQwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3Nw +LnNlY3RpZ28uY29tMIIBfQYKKwYBBAHWeQIEAgSCAW0EggFpAWcAdQDXbX0Q0af1 +d8LH6V/XAL/5gskzWmXh0LMBcxfAyMVpdwAAAZrg2UujAAAEAwBGMEQCIApVmNqo +gzJBNbcVXezO5sSvOFE5FaVZVz/eaqnCbG+2AiAT/A7XPtOYHwsE0wmTUBCTV/0l +bF7lk573b3rNtBvP3QB2AK9niDtXsE7dj6bZfvYuqOuBCsdxYPAkXlXWDC/nhYc6 +AAABmuDZTFoAAAQDAEcwRQIgODruJKtbjW1QJcQP7ARZAw5FgChfI599pJBA2bbQ +suICIQCskIUnzQCD6taycnQCN3zpu+rsz3Vd4AsMeFDM/cDJ5AB2AKyrMHBs6+yE +MfQT0vSRXxEeQiRDsfKmjE88KzunHgLDAAABmuDZS5kAAAQDAEcwRQIgD9IQCPTc +N88jbz5DUILwmDruTo411Ep5M2ZryNjBkywCIQCFwIyqGZEd+PiFv4l+5LOV3yDW +/zUuimFUoAJH5OIiNDBsBgNVHREEZTBjggtzc2xtYXRlLmNvbYIbKi5odHRwLWFw +cHJvdmFsLnNzbG1hdGUuY29tghFjZXJ0cy5zc2xtYXRlLmNvbYITY29uc29sZS5z +c2xtYXRlLmNvbYIPd3d3LnNzbG1hdGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBgQBR +Pjx14qo9PiYYEE1695CHdctA6up8L+n0MRapZcxALN/cetfGeoR00ZEH+7b1X7Ma +F9GGv1OtJXDoCySlAsdwFKHYtKhrUYRuQXdKGkTjdMzKO/+5kXeZIqgsCR10j8nr +Zq0Zcg2ply4j03/0y7+8ZNC1Erp4DB1Tq7ybgXnyURaNQTHsSkDoxMT/bWIrhGD0 +C8kN/ExkFvOBQlzdbuwo2d3v0zSM4mYmnqUhUYHprZllOziYgxIqjM/7mfnDkVAi +ov8yNJtn6EPt1wt6Oo3fC+Ft1T/kbSxeZbqWf3Zgbon5ijmNz+xqkb8br2+JdzM+ +8gEIqO6mNoMl0tayzb4a5KDaHxhczMGB3ggBwpVcdLtYBBa41thrgRP0VARqFTFG +IIkC9gPMjScf+uv9CQPsNk3kFI8vN4T3x4/g54N8Mc3M4JxvLaOsBj8dMeyq7v2p +1zE9WRngMUWuPgx0O94c0Pteumg/+pSGVeRapIuYZxXvkmLJ5wmwgYepix+cw1w= +-----END CERTIFICATE----- +` + +func TestComputeTBSHash(t *testing.T) { + certDER, err := parseCertificate([]byte(testCertPEM)) + if err != nil { + t.Fatalf("parseCertificate failed: %v", err) + } + tbsHash, err := computeTBSHash(certDER) + if err != nil { + t.Fatalf("computeTBSHash failed: %v", err) + } + if expected := [...]byte{0x3c, 0xf6, 0xb2, 0x44, 0xc2, 0x95, 0x85, 0xdb, 0xfb, 0xfd, 0x42, 0x0a, 0x6a, 0x4c, 0x62, 0xf7, 0x96, 0x8f, 0xa9, 0x05, 0xb4, 0xd6, 0xa4, 0xf5, 0x9d, 0x4d, 0x3b, 0xc9, 0xfa, 0xcb, 0x0c, 0xc8}; expected != tbsHash { + t.Fatalf("computeTBSHash returned %x; expected %x", tbsHash, expected) + } +} + +func TestCreateNotifiedMarker(t *testing.T) { + stateDir := t.TempDir() + + certDER, err := parseCertificate([]byte(testCertPEM)) + if err != nil { + t.Fatalf("parseCertificate failed: %v", err) + } + + tbsHash, err := computeTBSHash(certDER) + if err != nil { + t.Fatalf("computeTBSHash failed: %v", err) + } + + // First call should create the marker + notifiedPath, err := createNotifiedMarker(stateDir, tbsHash) + if err != nil { + t.Fatalf("createNotifiedMarker failed: %v", err) + } + + // Verify marker file exists + if !fileExists(notifiedPath) { + t.Fatalf("marker file does not exist: %s", notifiedPath) + } + + // Verify path structure is correct + tbsHex := hex.EncodeToString(tbsHash[:]) + expectedPath := filepath.Join(stateDir, "certs", tbsHex[0:2], "."+tbsHex+".notified") + if notifiedPath != expectedPath { + t.Fatalf("unexpected marker path: got %s, expected %s", notifiedPath, expectedPath) + } + + // Second call should succeed (idempotency) + notifiedPath2, err := createNotifiedMarker(stateDir, tbsHash) + if err != nil { + t.Fatalf("createNotifiedMarker second call failed: %v", err) + } + if notifiedPath != notifiedPath2 { + t.Fatalf("second call returned different path: got %s, expected %s", notifiedPath2, notifiedPath) + } +} + +func TestReadCertFile(t *testing.T) { + // Test reading from a file + tmpDir := t.TempDir() + certPath := filepath.Join(tmpDir, "cert.pem") + if err := os.WriteFile(certPath, []byte(testCertPEM), 0644); err != nil { + t.Fatalf("failed to write test cert: %v", err) + } + + certBytes, err := readCertFile(certPath) + if err != nil { + t.Fatalf("readCertFile failed: %v", err) + } + if !bytes.Equal(certBytes, []byte(testCertPEM)) { + t.Fatal("readCertFile returned different content") + } +} + +func TestFileExists(t *testing.T) { + tmpDir := t.TempDir() + + // Test with non-existent file + if fileExists(filepath.Join(tmpDir, "nonexistent")) { + t.Fatal("fileExists returned true for non-existent file") + } + + // Test with existing file + existingFile := filepath.Join(tmpDir, "existing") + if err := os.WriteFile(existingFile, []byte("test"), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + if !fileExists(existingFile) { + t.Fatal("fileExists returned false for existing file") + } +} + +func TestEndToEnd(t *testing.T) { + stateDir := t.TempDir() + + certDER, err := parseCertificate([]byte(testCertPEM)) + if err != nil { + t.Fatalf("parseCertificate failed: %v", err) + } + + tbsHash, err := computeTBSHash(certDER) + if err != nil { + t.Fatalf("computeTBSHash failed: %v", err) + } + + notifiedPath, err := createNotifiedMarker(stateDir, tbsHash) + if err != nil { + t.Fatalf("createNotifiedMarker failed: %v", err) + } + + // Verify the marker file structure matches what monitor/fsstate.go expects + tbsHex := hex.EncodeToString(tbsHash[:]) + expectedDir := filepath.Join(stateDir, "certs", tbsHex[0:2]) + expectedFile := filepath.Join(expectedDir, "."+tbsHex+".notified") + + if notifiedPath != expectedFile { + t.Fatalf("unexpected marker path: got %s, expected %s", notifiedPath, expectedFile) + } + + if !fileExists(expectedFile) { + t.Fatalf("marker file does not exist: %s", expectedFile) + } + + // Verify file is empty (as expected by certspotter) + stat, err := os.Stat(expectedFile) + if err != nil { + t.Fatalf("failed to stat marker file: %v", err) + } + if stat.Size() != 0 { + t.Fatalf("marker file should be empty, but has size %d", stat.Size()) + } +} diff --git a/third_party/certspotter/cmd/certspotter-metrics/.gitignore b/third_party/certspotter/cmd/certspotter-metrics/.gitignore new file mode 100644 index 0000000..795a03c --- /dev/null +++ b/third_party/certspotter/cmd/certspotter-metrics/.gitignore @@ -0,0 +1 @@ +/certspotter-metrics diff --git a/third_party/certspotter/cmd/certspotter-metrics/main.go b/third_party/certspotter/cmd/certspotter-metrics/main.go new file mode 100644 index 0000000..93add8b --- /dev/null +++ b/third_party/certspotter/cmd/certspotter-metrics/main.go @@ -0,0 +1,245 @@ +// Copyright (C) 2026 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "runtime/debug" + "slices" + "strconv" + "strings" + + "software.sslmate.com/src/certspotter/monitor" +) + +var programName = os.Args[0] +var Version = "unknown" +var Source = "unknown" + +func certspotterVersion() (string, string) { + if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") { + return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path + } else { + return Version, Source + } +} + +func homedir() string { + homedir, err := os.UserHomeDir() + if err != nil { + panic(fmt.Errorf("unable to determine home directory: %w", err)) + } + return homedir +} +func startedBySupervisor() bool { + return os.Getenv("SYSTEMD_EXEC_PID") == strconv.Itoa(os.Getpid()) +} +func defaultStateDir() string { + if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" { + return envVar + } else if envVar := os.Getenv("STATE_DIRECTORY"); envVar != "" && startedBySupervisor() { + return envVar + } else { + return filepath.Join(homedir(), ".certspotter") + } +} + +// logMetrics is the set of metric values for a single log. +type logMetrics struct { + logID monitor.LogID + + maxSTHExists bool + maxSTHSize uint64 + maxSTHTimestamp uint64 + downloadPosition uint64 + verifiedPosition uint64 + verifiedSTHExists bool + verifiedSTHTimestamp uint64 + unverifiedSTHs uint64 + malformedEntries uint64 + healthCheckFailures uint64 +} + +// metricFamily describes one of the metrics output by certspotter-metrics. +// All metrics are gauges labeled with the ID of the log they pertain to. +type metricFamily struct { + name string + help string + value func(*logMetrics) (uint64, bool) +} + +var metricFamilies = []metricFamily{ + { + name: "certspotter_log_max_sth_size", + help: "Tree size of largest known STH.", + value: func(m *logMetrics) (uint64, bool) { return m.maxSTHSize, m.maxSTHExists }, + }, + { + name: "certspotter_log_max_sth_timestamp", + help: "Timestamp (in milliseconds since the Epoch) of largest known STH.", + value: func(m *logMetrics) (uint64, bool) { return m.maxSTHTimestamp, m.maxSTHExists }, + }, + { + name: "certspotter_log_download_position", + help: "Number of entries downloaded from the log, including entries not yet verified.", + value: func(m *logMetrics) (uint64, bool) { return m.downloadPosition, true }, + }, + { + name: "certspotter_log_verified_position", + help: "Number of entries downloaded from the log and verified against an STH.", + value: func(m *logMetrics) (uint64, bool) { return m.verifiedPosition, true }, + }, + { + name: "certspotter_log_verified_sth_timestamp", + help: "Timestamp (in milliseconds since the Epoch) of the largest verified STH (the STH whose size is verified_position).", + value: func(m *logMetrics) (uint64, bool) { return m.verifiedSTHTimestamp, m.verifiedSTHExists }, + }, + { + name: "certspotter_log_unverified_sths", + help: "Number of STHs pending verification.", + value: func(m *logMetrics) (uint64, bool) { return m.unverifiedSTHs, true }, + }, + { + name: "certspotter_log_malformed_entries", + help: "Number of malformed entries recorded for the log.", + value: func(m *logMetrics) (uint64, bool) { return m.malformedEntries, true }, + }, + { + name: "certspotter_log_health_check_failures", + help: "Number of health check failures recorded for the log.", + value: func(m *logMetrics) (uint64, bool) { return m.healthCheckFailures, true }, + }, +} + +// gatherMetrics computes the metrics for every log that has a state directory. +// The returned slice is sorted by log ID. +func gatherMetrics(ctx context.Context, state *monitor.FilesystemState) ([]logMetrics, error) { + logIDs, err := state.ListLogs(ctx) + if err != nil { + return nil, fmt.Errorf("error listing logs in state directory: %w", err) + } + slices.SortFunc(logIDs, monitor.LogID.Compare) + + allMetrics := make([]logMetrics, 0, len(logIDs)) + for _, logID := range logIDs { + m := logMetrics{logID: logID} + + logState, err := state.LoadLogState(ctx, logID) + if err != nil { + return nil, fmt.Errorf("error loading state of log %s: %w", logID.Base64String(), err) + } + largestSTH, err := state.LoadLargestSTH(ctx, logID) + if err != nil { + return nil, fmt.Errorf("error loading STHs of log %s: %w", logID.Base64String(), err) + } + + if largestSTH != nil { + m.maxSTHExists = true + m.maxSTHSize = largestSTH.TreeSize + m.maxSTHTimestamp = largestSTH.Timestamp + } else if logState != nil && logState.VerifiedSTH != nil { + m.maxSTHExists = true + m.maxSTHSize = logState.VerifiedSTH.TreeSize + m.maxSTHTimestamp = logState.VerifiedSTH.Timestamp + } + + if logState != nil { + if logState.DownloadPosition != nil { + m.downloadPosition = logState.DownloadPosition.Size() + } + if logState.VerifiedPosition != nil { + m.verifiedPosition = logState.VerifiedPosition.Size() + } + if logState.VerifiedSTH != nil { + m.verifiedSTHExists = true + m.verifiedSTHTimestamp = logState.VerifiedSTH.Timestamp + } + } + + if unverifiedSTHs, err := state.CountSTHs(ctx, logID); err != nil { + return nil, fmt.Errorf("error counting STHs of log %s: %w", logID.Base64String(), err) + } else { + m.unverifiedSTHs = uint64(unverifiedSTHs) + } + if malformedEntries, err := state.CountMalformedEntries(ctx, logID); err != nil { + return nil, fmt.Errorf("error counting malformed entries of log %s: %w", logID.Base64String(), err) + } else { + m.malformedEntries = uint64(malformedEntries) + } + if healthCheckFailures, err := state.CountHealthCheckFailures(ctx, logID); err != nil { + return nil, fmt.Errorf("error counting health check failures of log %s: %w", logID.Base64String(), err) + } else { + m.healthCheckFailures = uint64(healthCheckFailures) + } + + allMetrics = append(allMetrics, m) + } + return allMetrics, nil +} + +// writeMetrics writes allMetrics to w in the Prometheus text exposition format. +func writeMetrics(w io.Writer, allMetrics []logMetrics) error { + buf := bufio.NewWriter(w) + for _, family := range metricFamilies { + if _, err := fmt.Fprintf(buf, "# HELP %s %s\n", family.name, family.help); err != nil { + return err + } + if _, err := fmt.Fprintf(buf, "# TYPE %s gauge\n", family.name); err != nil { + return err + } + for i := range allMetrics { + value, include := family.value(&allMetrics[i]) + if !include { + continue + } + if _, err := fmt.Fprintf(buf, "%s{log_id=\"%s\"} %d\n", family.name, allMetrics[i].logID.Base64String(), value); err != nil { + return err + } + } + } + return buf.Flush() +} + +func main() { + version, source := certspotterVersion() + + var flags struct { + stateDir string + version bool + } + flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory where certspotter stores state. This should be the same directory used by certspotter(8)") + flag.BoolVar(&flags.version, "version", false, "Print version and exit") + flag.Parse() + + if flags.version { + fmt.Fprintf(os.Stdout, "certspotter-metrics version %s (%s)\n", version, source) + os.Exit(0) + } + + state := &monitor.FilesystemState{ + StateDir: flags.stateDir, + } + + allMetrics, err := gatherMetrics(context.Background(), state) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) + os.Exit(1) + } + if err := writeMetrics(os.Stdout, allMetrics); err != nil { + fmt.Fprintf(os.Stderr, "%s: error writing metrics: %s\n", programName, err) + os.Exit(1) + } +} diff --git a/third_party/certspotter/cmd/certspotter/.gitignore b/third_party/certspotter/cmd/certspotter/.gitignore new file mode 100644 index 0000000..e4e7c7a --- /dev/null +++ b/third_party/certspotter/cmd/certspotter/.gitignore @@ -0,0 +1 @@ +/certspotter diff --git a/third_party/certspotter/cmd/certspotter/main.go b/third_party/certspotter/cmd/certspotter/main.go new file mode 100644 index 0000000..8534ce4 --- /dev/null +++ b/third_party/certspotter/cmd/certspotter/main.go @@ -0,0 +1,290 @@ +// Copyright (C) 2016, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io/fs" + "os" + "os/signal" + "path/filepath" + "runtime" + "runtime/debug" + "strconv" + "strings" + "syscall" + "time" + + "software.sslmate.com/src/certspotter/ctclient" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/monitor" +) + +var programName = os.Args[0] +var Version = "unknown" +var Source = "unknown" + +const defaultLogList = "https://loglist.certspotter.org/monitor.json" + +func certspotterVersion() (string, string) { + if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") { + return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path + } else { + return Version, Source + } +} + +func fileExists(filename string) bool { + _, err := os.Lstat(filename) + return err == nil +} +func homedir() string { + homedir, err := os.UserHomeDir() + if err != nil { + panic(fmt.Errorf("unable to determine home directory: %w", err)) + } + return homedir +} +func userCacheDir() string { + userCacheDir, err := os.UserCacheDir() + if err != nil { + panic(fmt.Errorf("unable to determine user cache directory: %w", err)) + } + return userCacheDir +} +func startedBySupervisor() bool { + return os.Getenv("SYSTEMD_EXEC_PID") == strconv.Itoa(os.Getpid()) +} +func defaultStateDir() string { + if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" { + return envVar + } else if envVar := os.Getenv("STATE_DIRECTORY"); envVar != "" && startedBySupervisor() { + return envVar + } else { + return filepath.Join(homedir(), ".certspotter") + } +} +func defaultConfigDir() string { + if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" { + return envVar + } else if envVar := os.Getenv("CONFIGURATION_DIRECTORY"); envVar != "" && startedBySupervisor() { + return envVar + } else { + return filepath.Join(homedir(), ".certspotter") + } +} +func defaultCacheDir() string { + if envVar := os.Getenv("CERTSPOTTER_CACHE_DIR"); envVar != "" { + return envVar + } else if envVar := os.Getenv("CACHE_DIRECTORY"); envVar != "" && startedBySupervisor() { + return envVar + } else { + return filepath.Join(userCacheDir(), "certspotter") + } +} +func defaultWatchListPath() string { + return filepath.Join(defaultConfigDir(), "watchlist") +} +func defaultWatchListPathIfExists() string { + if fileExists(defaultWatchListPath()) { + return defaultWatchListPath() + } else { + return "" + } +} +func defaultScriptDir() string { + return filepath.Join(defaultConfigDir(), "hooks.d") +} +func defaultEmailFile() string { + return filepath.Join(defaultConfigDir(), "email_recipients") +} + +func simplifyError(err error) error { + var pathErr *fs.PathError + if errors.As(err, &pathErr) { + return pathErr.Err + } + + return err +} + +func readWatchListFile(filename string) (monitor.WatchList, error) { + file, err := os.Open(filename) + if err != nil { + return nil, simplifyError(err) + } + defer file.Close() + return monitor.ReadWatchList(file) +} + +func readEmailFile(filename string) ([]string, error) { + file, err := os.Open(filename) + if err != nil { + return nil, simplifyError(err) + } + defer file.Close() + + var emails []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + emails = append(emails, line) + } + return emails, scanner.Err() +} + +func appendFunc(slice *[]string) func(string) error { + return func(value string) error { + *slice = append(*slice, value) + return nil + } +} + +func main() { + version, source := certspotterVersion() + + ctclient.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s; %s; +https://github.com/SSLMate/certspotter)", version, source, runtime.Version(), runtime.GOOS, runtime.GOARCH) + loglist.UserAgent = ctclient.UserAgent + + var flags struct { + email []string + healthcheck time.Duration + logs string + noSave bool + script string + startAtEnd bool + stateDir string + stdout bool + verbose bool + version bool + watchlist string + } + flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email)) + flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check") + flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor") + flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory") + flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered") + flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)") + flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates") + flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout") + flag.BoolVar(&flags.verbose, "verbose", false, "Print detailed information about certspotter's operation to stderr") + flag.BoolVar(&flags.version, "version", false, "Print version and exit") + flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch") + flag.Parse() + + if flags.version { + fmt.Fprintf(os.Stdout, "certspotter version %s (%s)\n", version, source) + if flags.verbose { + if buildinfo, ok := debug.ReadBuildInfo(); ok { + fmt.Fprint(os.Stdout, buildinfo) + } else { + fmt.Fprint(os.Stdout, "build info not available\n") + } + } + os.Exit(0) + } + if flags.watchlist == "" { + fmt.Fprintf(os.Stderr, "%s: watch list not found: please create %s or specify alternative path using -watchlist\n", programName, defaultWatchListPath()) + os.Exit(2) + } + + fsstate := &monitor.FilesystemState{ + StateDir: flags.stateDir, + CacheDir: defaultCacheDir(), + SaveCerts: !flags.noSave, + Script: flags.script, + ScriptDir: defaultScriptDir(), + Email: flags.email, + Stdout: flags.stdout, + } + config := &monitor.Config{ + LogListSource: flags.logs, + State: fsstate, + StartAtEnd: flags.startAtEnd, + Verbose: flags.verbose, + HealthCheckInterval: flags.healthcheck, + } + + emailFileExists := false + if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil { + emailFileExists = true + fsstate.Email = append(fsstate.Email, emailRecipients...) + } else if !errors.Is(err, fs.ErrNotExist) { + fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err) + os.Exit(1) + } + + if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false { + fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName) + fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n") + fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile()) + fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir) + fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n") + fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n") + fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n") + os.Exit(2) + } + + if flags.watchlist == "-" { + watchlist, err := monitor.ReadWatchList(os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err) + os.Exit(1) + } + config.WatchList = watchlist + } else { + watchlist, err := readWatchListFile(flags.watchlist) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err) + os.Exit(1) + } + config.WatchList = watchlist + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + go func() { + ticker := time.NewTicker(24*time.Hour) + defer ticker.Stop() + for { + fsstate.PruneOldErrors() + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + + if startedBySupervisor() { + // Prevent any process that we spawn from erroneously writing to our directories + os.Unsetenv("CACHE_DIRECTORY") + os.Unsetenv("CONFIGURATION_DIRECTORY") + os.Unsetenv("STATE_DIRECTORY") + } + + if err := monitor.Run(ctx, config); ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) { + if flags.verbose { + fmt.Fprintf(os.Stderr, "%s: exiting due to SIGINT or SIGTERM\n", programName) + } + os.Exit(0) + } else { + fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) + os.Exit(1) + } +} diff --git a/third_party/certspotter/ctclient/client.go b/third_party/certspotter/ctclient/client.go new file mode 100644 index 0000000..ccf7edc --- /dev/null +++ b/third_party/certspotter/ctclient/client.go @@ -0,0 +1,204 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +// Package ctclient implements a client for monitoring RFC6962 and static-ct-api Certificate Transparency logs +package ctclient + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "time" + + "software.sslmate.com/src/certspotter/cttypes" +) + +var UserAgent = "" + +// Create an HTTP client suitable for communicating with CT logs. dialContext, if non-nil, is used for dialing. +func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn, error)) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + // We have to disable TLS certificate validation because because several logs + // (WoSign, StartCom, GDCA) use certificates that are not widely trusted. + // Since we verify that every response we receive from the log is signed + // by the log's CT public key (either directly, or indirectly via the Merkle Tree), + // TLS certificate validation is not actually necessary. (We don't want to manage + // our own trust store because that adds undesired complexity and would require + // updating should a log ever change to a different CA.) + InsecureSkipVerify: true, + }, + DialContext: dialContext, + ForceAttemptHTTP2: true, + }, + CheckRedirect: func(*http.Request, []*http.Request) error { + return errors.New("redirects not followed") + }, + Timeout: 60 * time.Second, + } +} + +var defaultHTTPClient = NewHTTPClient(nil) + +// maxResponseBytes is a generous upper bound on the size of an HTTP response +// body that we are willing to buffer in memory. +const maxResponseBytes = 64 << 20 // 64 MiB + +// maxErrorResponseBytes is longest error response that we'll read +const maxErrorResponseBytes = 8 << 10 // 8 KiB + +// readResponseBody reads a response body, returning an error if the +// body is larger than maxBytes. +func readResponseBody(r io.Reader, maxBytes int64) ([]byte, error) { + body, err := io.ReadAll(io.LimitReader(r, maxBytes+1)) + if err != nil { + return nil, fmt.Errorf("error reading response: %w", err) + } + if int64(len(body)) > maxBytes { + return nil, fmt.Errorf("response body exceeds maximum allowed size of %d bytes", maxBytes) + } + return body, nil +} + +func get(ctx context.Context, httpClient *http.Client, fullURL string) (io.ReadCloser, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil) + if err != nil { + return nil, err + } + request.Header.Set("User-Agent", UserAgent) + + if httpClient == nil { + httpClient = defaultHTTPClient + } + + response, err := httpClient.Do(request) + if err != nil { + return nil, err + } + + if response.StatusCode != 200 { + defer response.Body.Close() + responseBody, _ := io.ReadAll(io.LimitReader(response.Body, maxErrorResponseBytes)) + return nil, fmt.Errorf("Get %q: %s (%s)", fullURL, response.Status, formatResponseBody(responseBody)) + } + + return response.Body, nil +} + +func getBytes(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte, error) { + body, err := get(ctx, httpClient, fullURL) + if err != nil { + return nil, err + } + defer body.Close() + bodyBytes, err := readResponseBody(body, maxResponseBytes) + if err != nil { + return nil, fmt.Errorf("Get %q: %w", fullURL, err) + } + return bodyBytes, nil +} + +func getJSON(ctx context.Context, httpClient *http.Client, fullURL string, response any) error { + responseBytes, err := getBytes(ctx, httpClient, fullURL) + if err != nil { + return err + } + if err := json.Unmarshal(responseBytes, response); err != nil { + return fmt.Errorf("Get %q: error parsing response JSON: %w", fullURL, err) + } + return nil +} + +func getRoots(ctx context.Context, httpClient *http.Client, logURL *url.URL) ([][]byte, error) { + fullURL := logURL.JoinPath("/ct/v1/get-roots").String() + var parsedResponse struct { + Certificates [][]byte `json:"certificates"` + } + if err := getJSON(ctx, httpClient, fullURL, &parsedResponse); err != nil { + return nil, err + } + return parsedResponse.Certificates, nil +} + +func addChainOrPreChain(ctx context.Context, httpClient *http.Client, logURL *url.URL, isPreChain bool, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) { + var endpoint string + if isPreChain { + endpoint = "/ct/v1/add-pre-chain" + } else { + endpoint = "/ct/v1/add-chain" + } + fullURL := logURL.JoinPath(endpoint).String() + + requestBody, err := json.Marshal(struct { + Chain [][]byte `json:"chain"` + }{ + Chain: chain, + }) + if err != nil { + return nil, err + } + + request, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(requestBody)) + if err != nil { + return nil, err + } + request.Header.Set("User-Agent", UserAgent) + request.Header.Set("Content-Type", "application/json") + + if httpClient == nil { + httpClient = defaultHTTPClient + } + + response, err := httpClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + responseBody, err := readResponseBody(response.Body, maxResponseBytes) + if err != nil { + return nil, fmt.Errorf("Post %q: %w", fullURL, err) + } + + if response.StatusCode != 200 { + return nil, fmt.Errorf("Post %q: %s (%s)", fullURL, response.Status, formatResponseBody(responseBody)) + } + + sct := new(cttypes.SignedCertificateTimestamp) + if err := json.Unmarshal(responseBody, sct); err != nil { + return nil, fmt.Errorf("Post %q: error parsing response JSON: %w", fullURL, err) + } + + return sct, nil +} + +func formatResponseBody(body []byte) string { + const maxLen = 200 + body = bytes.TrimSpace(body) + if len(body) > maxLen { + return strconv.QuoteToASCII(string(body[:maxLen])) + "..." + } else { + return strconv.QuoteToASCII(string(body)) + } +} diff --git a/third_party/certspotter/ctclient/client_test.go b/third_party/certspotter/ctclient/client_test.go new file mode 100644 index 0000000..590b9e9 --- /dev/null +++ b/third_party/certspotter/ctclient/client_test.go @@ -0,0 +1,77 @@ +// Copyright (C) 2026 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctclient + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestReadResponseBody(t *testing.T) { + // under the limit + if body, err := readResponseBody(strings.NewReader("hello"), 10); err != nil { + t.Errorf("unexpected error under limit: %v", err) + } else if string(body) != "hello" { + t.Errorf("got %q, want %q", body, "hello") + } + + // exactly at the limit + if body, err := readResponseBody(strings.NewReader("0123456789"), 10); err != nil { + t.Errorf("unexpected error at exact limit: %v", err) + } else if len(body) != 10 { + t.Errorf("got %d bytes, want 10", len(body)) + } + + // one byte over the limit + if _, err := readResponseBody(strings.NewReader("0123456789X"), 10); err == nil { + t.Errorf("expected error for oversize body, got nil") + } +} + +// Confirm the cap is actually wired into getBytes(): a body larger than the limit is +// rejected with an error instead of being buffered in full. +func TestGetEnforcesResponseLimit(t *testing.T) { + const bodyLen = maxResponseBytes + 1024 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buf := make([]byte, 32*1024) + for written := 0; written < bodyLen; written += len(buf) { + if _, err := w.Write(buf); err != nil { + return + } + } + })) + defer server.Close() + + if _, err := getBytes(context.Background(), server.Client(), server.URL); err == nil { + t.Errorf("getBytes() accepted an over-limit response body; expected an error") + } else if !strings.Contains(err.Error(), "maximum allowed size") { + t.Errorf("unexpected error: %v", err) + } +} + +// A normal-sized response still round-trips correctly. +func TestGetReadsNormalBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "tile-data") + })) + defer server.Close() + + body, err := getBytes(context.Background(), server.Client(), server.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(body) != "tile-data" { + t.Errorf("got %q, want %q", body, "tile-data") + } +} diff --git a/third_party/certspotter/ctclient/log.go b/third_party/certspotter/ctclient/log.go new file mode 100644 index 0000000..d18d098 --- /dev/null +++ b/third_party/certspotter/ctclient/log.go @@ -0,0 +1,59 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctclient + +import ( + "context" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/merkletree" +) + +type WritableLog interface { + AddChain(context.Context, [][]byte) (*cttypes.SignedCertificateTimestamp, error) + AddPreChain(context.Context, [][]byte) (*cttypes.SignedCertificateTimestamp, error) + GetRoots(context.Context) ([][]byte, error) +} + +type Log interface { + GetSTH(context.Context) (*cttypes.SignedTreeHead, string, error) + GetRoots(context.Context) ([][]byte, error) + GetEntries(ctx context.Context, startInclusive, endInclusive uint64) ([]Entry, error) + ReconstructTree(context.Context, *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) +} + +// IssuerGetter represents a source of issuer certificates. +// +// If a [Log] also implements IssuerGetter, then it is mandatory to provide +// an IssuerGetter when using [Entry]s returned by the [Log]. The IssuerGetter +// may be the Log itself, or your own implementation which retrieves issuers +// from a different source, such as a cache. +// +// If a Log doesn't implement IssuerGetter, then you may pass a nil IssuerGetter +// when using the Log's Entrys. +type IssuerGetter interface { + GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) +} + +type Entry interface { + LeafInput() []byte + + // Returns error from IssuerGetter, otherwise infallible + ExtraData(context.Context, IssuerGetter) ([]byte, error) + + // Returns an error if this is not a well-formed precert entry + Precertificate() (cttypes.ASN1Cert, error) + + // Returns an error if this is not a well-formed x509 or precert entry + ChainFingerprints() ([][32]byte, error) + + // Returns an error if this is not a well-formed x509 or precert entry, or if IssuerGetter failed + GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error) +} diff --git a/third_party/certspotter/ctclient/rfc6962.go b/third_party/certspotter/ctclient/rfc6962.go new file mode 100644 index 0000000..df0bb73 --- /dev/null +++ b/third_party/certspotter/ctclient/rfc6962.go @@ -0,0 +1,272 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctclient + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "slices" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/merkletree" +) + +type RFC6962Log struct { + URL *url.URL + HTTPClient *http.Client // nil to use default client +} + +type RFC6962LogEntry struct { + Leaf_input []byte `json:"leaf_input"` + Extra_data []byte `json:"extra_data"` +} + +func (ctlog *RFC6962Log) AddChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) { + return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.URL, false, chain) +} + +func (ctlog *RFC6962Log) AddPreChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) { + return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.URL, true, chain) +} + +func (ctlog *RFC6962Log) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) { + fullURL := ctlog.URL.JoinPath("/ct/v1/get-sth").String() + sth := new(cttypes.SignedTreeHead) + if err := getJSON(ctx, ctlog.HTTPClient, fullURL, sth); err != nil { + return nil, fullURL, err + } + return sth, fullURL, nil +} + +func (ctlog *RFC6962Log) GetRoots(ctx context.Context) ([][]byte, error) { + return getRoots(ctx, ctlog.HTTPClient, ctlog.URL) +} + +func (ctlog *RFC6962Log) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]RFC6962LogEntry, error) { + fullURL := ctlog.URL.JoinPath("/ct/v1/get-entries").String() + fullURL += fmt.Sprintf("?start=%d&end=%d", startInclusive, endInclusive) + + body, err := get(ctx, ctlog.HTTPClient, fullURL) + if err != nil { + return nil, err + } + defer body.Close() + + limitedBody := io.LimitReader(body, maxResponseBytes) + dec := json.NewDecoder(limitedBody) + if tok, err := dec.Token(); err != nil { + return nil, fmt.Errorf("Get %q: error reading JSON object token: %w", fullURL, err) + } else if tok != json.Delim('{') { + return nil, fmt.Errorf("Get %q: not a JSON object", fullURL) + } + for { + if !dec.More() { + return nil, fmt.Errorf("Get %q: JSON object missing entries", fullURL) + } + tok, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("Get %q: error reading JSON field name: %w", fullURL, err) + } + if tok == "entries" { + break + } + var rawMessage json.RawMessage + if err := dec.Decode(&rawMessage); err != nil { + return nil, fmt.Errorf("Get %q: error skipping unwanted JSON field: %w", fullURL, err) + } + } + if tok, err := dec.Token(); err != nil { + return nil, fmt.Errorf("Get %q: error reading JSON array token: %w", fullURL, err) + } else if tok != json.Delim('[') { + return nil, fmt.Errorf("Get %q: entries is not a JSON array", fullURL) + } + + var entries []RFC6962LogEntry + for dec.More() { + if uint64(len(entries)) == endInclusive-startInclusive+1 { + return entries, fmt.Errorf("Get %q: extraneous entries returned", fullURL) + } + var entry RFC6962LogEntry + if err := dec.Decode(&entry); err != nil { + return entries, fmt.Errorf("Get %q: error reading JSON entry: %w", fullURL, err) + } + entries = append(entries, entry) + } + if tok, err := dec.Token(); err != nil { + return entries, fmt.Errorf("Get %q: error reading end of JSON array: %w", fullURL, err) + } else if tok != json.Delim(']') { + return entries, fmt.Errorf("Get %q: entries array not terminated", fullURL) + } + if len(entries) == 0 { + return entries, fmt.Errorf("Get %q: zero entries returned", fullURL) + } + io.Copy(io.Discard, limitedBody) // So the HTTP connection can be reused + return entries, nil +} + +func (ctlog *RFC6962Log) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) { + nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive) + var entries []Entry + if nativeEntries != nil { + entries = make([]Entry, len(nativeEntries)) + for i := range nativeEntries { + entries[i] = &nativeEntries[i] + } + } + return entries, err +} + +type entryAndProofResponse struct { + LeafInput []byte `json:"leaf_input"` + ExtraData []byte `json:"extra_data"` + AuditPath []merkletree.Hash `json:"audit_path"` +} + +func (ctlog *RFC6962Log) getEntryAndProof(ctx context.Context, leafIndex uint64, treeSize uint64) (*entryAndProofResponse, error) { + fullURL := ctlog.URL.JoinPath("/ct/v1/get-entry-and-proof").String() + fullURL += fmt.Sprintf("?leaf_index=%d&tree_size=%d", leafIndex, treeSize) + response := new(entryAndProofResponse) + if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil { + return nil, err + } + return response, nil +} + +type proofResponse struct { + LeafIndex uint64 `json:"leaf_index"` + AuditPath []merkletree.Hash `json:"audit_path"` +} + +func (ctlog *RFC6962Log) getProofByHash(ctx context.Context, hash *merkletree.Hash, treeSize uint64) (*proofResponse, error) { + fullURL := ctlog.URL.JoinPath("/ct/v1/get-proof-by-hash").String() + fullURL += fmt.Sprintf("?hash=%s&tree_size=%d", url.QueryEscape(hash.Base64String()), treeSize) + response := new(proofResponse) + if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil { + return nil, err + } + return response, nil +} + +func (ctlog *RFC6962Log) reconstructTree(ctx context.Context, treeSize uint64) (*merkletree.CollapsedTree, error) { + if treeSize == 0 { + return new(merkletree.CollapsedTree), nil + } + if entryAndProof, err := ctlog.getEntryAndProof(ctx, treeSize-1, treeSize); err == nil { + tree := new(merkletree.CollapsedTree) + slices.Reverse(entryAndProof.AuditPath) + if err := tree.Init(entryAndProof.AuditPath, treeSize-1); err != nil { + return nil, fmt.Errorf("log returned invalid audit proof for entry %d to STH %d: %w", treeSize-1, treeSize, err) + } + tree.Add(merkletree.HashLeaf(entryAndProof.LeafInput)) + return tree, nil + } + entries, err := ctlog.getEntries(ctx, treeSize-1, treeSize-1) + if err != nil { + return nil, err + } + leafHash := merkletree.HashLeaf(entries[0].Leaf_input) + tree := new(merkletree.CollapsedTree) + if treeSize > 1 { + response, err := ctlog.getProofByHash(ctx, &leafHash, treeSize) + if err != nil { + return nil, err + } + if response.LeafIndex != treeSize-1 { + // This can happen if the leaf hash is present in the tree in more than one place. Unfortunately, we can't reconstruct when tree if this happens. Fortunately, this is really unlikely, and most logs support get-entry-and-proof anyways. + return nil, fmt.Errorf("unable to reconstruct tree because leaf hash %s is present in tree at more than one index (need proof for index %d but get-proof-by-hash returned proof for index %d)", leafHash.Base64String(), treeSize-1, response.LeafIndex) + } + slices.Reverse(response.AuditPath) + if err := tree.Init(response.AuditPath, treeSize-1); err != nil { + return nil, fmt.Errorf("log returned invalid audit proof for hash %s to STH %d: %w", leafHash.Base64String(), treeSize, err) + } + } + tree.Add(leafHash) + return tree, nil +} + +func (ctlog *RFC6962Log) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) { + tree, err := ctlog.reconstructTree(ctx, sth.TreeSize) + if err != nil { + return nil, err + } + + if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash { + return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize) + } + + return tree, nil +} + +func (entry *RFC6962LogEntry) isX509() bool { + return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 0 +} + +func (entry *RFC6962LogEntry) isPrecert() bool { + return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 1 +} + +func (entry *RFC6962LogEntry) LeafInput() []byte { + return entry.Leaf_input +} + +func (entry *RFC6962LogEntry) ExtraData(context.Context, IssuerGetter) ([]byte, error) { + return entry.Extra_data, nil +} + +func (entry *RFC6962LogEntry) Precertificate() (cttypes.ASN1Cert, error) { + if !entry.isPrecert() { + return nil, fmt.Errorf("not a precertificate entry") + } + extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data) + if err != nil { + return nil, fmt.Errorf("error parsing extra_data: %w", err) + } + return extraData.PreCertificate, nil +} + +func (entry *RFC6962LogEntry) ChainFingerprints() ([][32]byte, error) { + chain, err := entry.parseChain() + if err != nil { + return nil, err + } + fingerprints := make([][32]byte, len(chain)) + for i := range chain { + fingerprints[i] = sha256.Sum256(chain[i]) + } + return fingerprints, nil +} + +func (entry *RFC6962LogEntry) GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error) { + return entry.parseChain() +} + +func (entry *RFC6962LogEntry) parseChain() (cttypes.ASN1CertChain, error) { + switch { + case entry.isX509(): + extraData, err := cttypes.ParseExtraDataForX509Entry(entry.Extra_data) + if err != nil { + return nil, fmt.Errorf("error parsing extra_data for X509 entry: %w", err) + } + return extraData, nil + case entry.isPrecert(): + extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data) + if err != nil { + return nil, fmt.Errorf("error parsing extra_data for precert entry: %w", err) + } + return extraData.PrecertificateChain, nil + default: + return nil, fmt.Errorf("unknown entry type") + } +} diff --git a/third_party/certspotter/ctclient/static.go b/third_party/certspotter/ctclient/static.go new file mode 100644 index 0000000..782624e --- /dev/null +++ b/third_party/certspotter/ctclient/static.go @@ -0,0 +1,401 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/url" + "slices" + "strconv" + "sync" + + "golang.org/x/crypto/cryptobyte" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/merkletree" +) + +const ( + staticTileHeight = 8 + StaticTileWidth = 1 << staticTileHeight +) + +func staticSubtreeSize(level uint64) uint64 { return 1 << (level * staticTileHeight) } + +type StaticLog struct { + SubmissionURL *url.URL + MonitoringURL *url.URL + ID cttypes.LogID + HTTPClient *http.Client // nil to use default client +} + +type StaticLogEntry struct { + timestampedEntry []byte + precertificate []byte // nil iff x509 entry; non-nil iff precert entry + chain [][32]byte +} + +func (ctlog *StaticLog) AddChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) { + return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.SubmissionURL, false, chain) +} + +func (ctlog *StaticLog) AddPreChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) { + return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.SubmissionURL, true, chain) +} + +func (ctlog *StaticLog) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) { + fullURL := ctlog.MonitoringURL.JoinPath("/checkpoint").String() + responseBody, err := getBytes(ctx, ctlog.HTTPClient, fullURL) + if err != nil { + return nil, fullURL, err + } + sth, err := cttypes.ParseCheckpoint(responseBody, ctlog.ID) + if err != nil { + return nil, fullURL, err + } + return sth, fullURL, nil +} + +func (ctlog *StaticLog) GetRoots(ctx context.Context) ([][]byte, error) { + return getRoots(ctx, ctlog.HTTPClient, ctlog.SubmissionURL) +} + +func (ctlog *StaticLog) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]StaticLogEntry, error) { + var ( + tile = startInclusive / StaticTileWidth + skip = startInclusive % StaticTileWidth + tileWidth = min(StaticTileWidth, endInclusive+1-tile*StaticTileWidth) + numEntries = tileWidth - skip + ) + + data, err := ctlog.getDataTile(ctx, tile, tileWidth) + if err != nil { + return nil, err + } + var skippedEntry StaticLogEntry + for i := range skip { + if rest, err := skippedEntry.parse(data); err != nil { + return nil, fmt.Errorf("error parsing skipped entry %d in tile %d: %w", i, tile, err) + } else { + data = rest + } + } + entries := make([]StaticLogEntry, numEntries) + for i := range numEntries { + if rest, err := entries[i].parse(data); err != nil { + return nil, fmt.Errorf("error parsing entry %d in tile %d: %w", skip+i, tile, err) + } else { + data = rest + } + } + return entries, nil +} + +func (ctlog *StaticLog) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) { + nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive) + if err != nil { + return nil, err + } + entries := make([]Entry, len(nativeEntries)) + for i := range nativeEntries { + entries[i] = &nativeEntries[i] + } + return entries, nil +} + +func (ctlog *StaticLog) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) { + type job struct { + level uint64 + offset uint64 + width uint64 + } + var jobs []job + for level, size := uint64(0), sth.TreeSize; size > 0; level++ { + fullTiles := size / StaticTileWidth + remainder := size % StaticTileWidth + size = fullTiles + if remainder > 0 { + jobs = append(jobs, job{ + level: level, + offset: fullTiles, + width: remainder, + }) + } + } + trees := make([]*merkletree.CollapsedTree, len(jobs)) + errs := make([]error, len(jobs)) + + var wg sync.WaitGroup + for i, job := range jobs { + wg.Add(1) + go func() { + trees[i], errs[i] = ctlog.getTileCollapsedTree(ctx, job.level, job.offset, job.width) + wg.Done() + }() + } + wg.Wait() + + if err := errors.Join(errs...); err != nil { + return nil, err + } + + tree := new(merkletree.CollapsedTree) + for _, subtree := range slices.Backward(trees) { + if err := tree.Append(*subtree); err != nil { + panic(err) + } + } + + if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash { + return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize) + } + + return tree, nil +} + +func (ctlog *StaticLog) getDataTile(ctx context.Context, tile uint64, width uint64) ([]byte, error) { + if width == 0 || width > StaticTileWidth { + panic("width is out of range") + } + var partialErr error + if width < StaticTileWidth { + fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, width)).String() + if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil { + partialErr = err + } else { + return data, nil + } + } + + fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, 0)).String() + if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil { + if partialErr != nil { + return nil, partialErr + } else { + return nil, err + } + } else { + return data, nil + } +} + +// returned slice is numHashes*merkletree.HashLen bytes long +func (ctlog *StaticLog) getTile(ctx context.Context, level uint64, tile uint64, numHashes uint64) ([]byte, error) { + if numHashes == 0 || numHashes > StaticTileWidth { + panic("numHashes is out of range") + } + + var partialErr error + if numHashes < StaticTileWidth { + fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, numHashes)).String() + if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil { + partialErr = err + } else if expectedLen := merkletree.HashLen * int(numHashes); len(data) != expectedLen { + return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen) + } else { + return data, nil + } + } + + fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, 0)).String() + if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil { + if partialErr != nil { + return nil, partialErr + } else { + return nil, err + } + } else if expectedLen := merkletree.HashLen * StaticTileWidth; len(data) != expectedLen { + return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen) + } else { + desiredLen := merkletree.HashLen * int(numHashes) + return data[:desiredLen], nil + } +} + +func (ctlog *StaticLog) getTileCollapsedTree(ctx context.Context, level uint64, tile uint64, numHashes uint64) (*merkletree.CollapsedTree, error) { + data, err := ctlog.getTile(ctx, level, tile, numHashes) + if err != nil { + return nil, err + } + subtreeSize := staticSubtreeSize(level) + offset := staticSubtreeSize(level+1) * tile + + tree := new(merkletree.CollapsedTree) + if err := tree.InitSubtree(offset, nil, 0); err != nil { + panic(err) + } + for i := uint64(0); i < numHashes; i++ { + hash := (merkletree.Hash)(data[i*merkletree.HashLen : (i+1)*merkletree.HashLen]) + var subtree merkletree.CollapsedTree + if err := subtree.InitSubtree(offset+i*subtreeSize, []merkletree.Hash{hash}, subtreeSize); err != nil { + panic(err) + } + if err := tree.Append(subtree); err != nil { + panic(err) + } + } + return tree, nil +} + +func (ctlog *StaticLog) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) { + fullURL := ctlog.MonitoringURL.JoinPath("/issuer/" + hex.EncodeToString(fingerprint[:])).String() + data, err := getBytes(ctx, ctlog.HTTPClient, fullURL) + if err != nil { + return nil, err + } + if gotFingerprint := sha256.Sum256(data); gotFingerprint != *fingerprint { + return nil, fmt.Errorf("%s returned incorrect data with fingerprint %x", fullURL, gotFingerprint[:]) + } + return data, nil +} + +func (entry *StaticLogEntry) parse(input []byte) ([]byte, error) { + var skipped cryptobyte.String + str := cryptobyte.String(input) + + // TimestampedEntry.timestamp + if !str.Skip(8) { + return nil, fmt.Errorf("error reading timestamp") + } + // TimestampedEntry.entry_type + var entryType uint16 + if !str.ReadUint16(&entryType) { + return nil, fmt.Errorf("error reading entry type") + } + // TimestampedEntry.signed_entry + if entryType == 0 { + if !str.ReadUint24LengthPrefixed(&skipped) { + return nil, fmt.Errorf("error reading certificate") + } + } else if entryType == 1 { + if !str.Skip(32) { + return nil, fmt.Errorf("error reading issuer_key_hash") + } + if !str.ReadUint24LengthPrefixed(&skipped) { + return nil, fmt.Errorf("error reading tbs_certificate") + } + } else { + return nil, fmt.Errorf("invalid entry type %d", entryType) + } + + // TimestampedEntry.extensions + if !str.ReadUint16LengthPrefixed(&skipped) { + return nil, fmt.Errorf("error reading extensions") + } + + timestampedEntryLen := len(input) - len(str) + entry.timestampedEntry = input[:timestampedEntryLen] + + // precertificate + if entryType == 1 { + var precertificate cryptobyte.String + if !str.ReadUint24LengthPrefixed(&precertificate) { + return nil, fmt.Errorf("error reading precertificate") + } + entry.precertificate = precertificate + } else { + entry.precertificate = nil + } + + // certificate_chain + var chainBytes cryptobyte.String + if !str.ReadUint16LengthPrefixed(&chainBytes) { + return nil, fmt.Errorf("error reading certificate_chain") + } + entry.chain = make([][32]byte, 0, len(chainBytes)/32) + for !chainBytes.Empty() { + var fingerprint [32]byte + if !chainBytes.CopyBytes(fingerprint[:]) { + return nil, fmt.Errorf("error reading fingerprint in certificate_chain") + } + entry.chain = append(entry.chain, fingerprint) + } + + return str, nil +} + +func (entry *StaticLogEntry) LeafInput() []byte { + return append([]byte{0, 0}, entry.timestampedEntry...) +} + +func (entry *StaticLogEntry) ExtraData(ctx context.Context, issuerGetter IssuerGetter) ([]byte, error) { + b := cryptobyte.NewBuilder(nil) + if entry.precertificate != nil { + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(entry.precertificate) + }) + } + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + for _, fingerprint := range entry.chain { + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + cert, err := issuerGetter.GetIssuer(ctx, &fingerprint) + if err != nil { + panic(cryptobyte.BuildError{Err: fmt.Errorf("error getting issuer %x: %w", fingerprint, err)}) + } + b.AddBytes(cert) + }) + } + }) + return b.Bytes() +} + +func (entry *StaticLogEntry) Precertificate() (cttypes.ASN1Cert, error) { + if entry.precertificate == nil { + return nil, fmt.Errorf("not a precertificate entry") + } + return entry.precertificate, nil +} + +func (entry *StaticLogEntry) ChainFingerprints() ([][32]byte, error) { + return entry.chain, nil +} + +func (entry *StaticLogEntry) GetChain(ctx context.Context, issuerGetter IssuerGetter) (cttypes.ASN1CertChain, error) { + var ( + chain = make(cttypes.ASN1CertChain, len(entry.chain)) + errs = make([]error, len(entry.chain)) + ) + var wg sync.WaitGroup + for i, fingerprint := range entry.chain { + wg.Add(1) + go func() { + defer wg.Done() + chain[i], errs[i] = issuerGetter.GetIssuer(ctx, &fingerprint) + }() + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return nil, err + } + return chain, nil +} + +func formatTilePath(level string, tile uint64, partial uint64) string { + path := "tile/" + level + "/" + formatTileIndex(tile) + if partial != 0 { + path += fmt.Sprintf(".p/%d", partial) + } + return path +} + +func formatTileIndex(tile uint64) string { + const base = 1000 + str := fmt.Sprintf("%03d", tile%base) + for tile >= base { + tile = tile / base + str = fmt.Sprintf("x%03d/%s", tile%base, str) + } + return str +} diff --git a/third_party/certspotter/ctclient/static_test.go b/third_party/certspotter/ctclient/static_test.go new file mode 100644 index 0000000..af38906 --- /dev/null +++ b/third_party/certspotter/ctclient/static_test.go @@ -0,0 +1,38 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctclient + +import ( + "testing" +) + +func TestFormatTileIndex(t *testing.T) { + tests := []struct { + in uint64 + out string + }{ + {0, "000"}, + {1, "001"}, + {12, "012"}, + {105, "105"}, + {1000, "x001/000"}, + {1050, "x001/050"}, + {52123, "x052/123"}, + {999001, "x999/001"}, + {1999001, "x001/x999/001"}, + {15999001, "x015/x999/001"}, + } + for i, test := range tests { + result := formatTileIndex(test.in) + if result != test.out { + t.Errorf("#%d: formatTileIndex(%q) = %q, want %q", i, test.in, result, test.out) + } + } +} diff --git a/third_party/certspotter/ctcrypto/key.go b/third_party/certspotter/ctcrypto/key.go new file mode 100644 index 0000000..23bd3b1 --- /dev/null +++ b/third_party/certspotter/ctcrypto/key.go @@ -0,0 +1,67 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctcrypto + +import ( + "bytes" + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/x509" + "fmt" + + "software.sslmate.com/src/certspotter/tlstypes" +) + +type PublicKey []byte + +func (key PublicKey) Verify(input SignatureInput, signature tlstypes.DigitallySigned) error { + parsedKey, err := x509.ParsePKIXPublicKey(key) + if err != nil { + return fmt.Errorf("error parsing log key: %w", err) + } + switch key := parsedKey.(type) { + case *rsa.PublicKey: + if signature.Algorithm.Signature != tlstypes.RSA { + return fmt.Errorf("log key is RSA but this is not an RSA signature") + } + if signature.Algorithm.Hash != tlstypes.SHA256 { + return fmt.Errorf("unsupported hash algorithm %v (only SHA-256 is allowed in CT)", signature.Algorithm.Hash) + } + if rsa.VerifyPKCS1v15((*rsa.PublicKey)(key), crypto.SHA256, input[:], signature.Signature) != nil { + return fmt.Errorf("RSA signature is incorrect") + } + return nil + + case *ecdsa.PublicKey: + if signature.Algorithm.Signature != tlstypes.ECDSA { + return fmt.Errorf("log key is ECDSA but this is not an ECDSA signature") + } + if signature.Algorithm.Hash != tlstypes.SHA256 { + return fmt.Errorf("unsupported hash algorithm %v (only SHA-256 is allowed in CT)", signature.Algorithm.Hash) + } + if !ecdsa.VerifyASN1((*ecdsa.PublicKey)(key), input[:], signature.Signature) { + return fmt.Errorf("ECDSA signature is incorrect") + } + + default: + return fmt.Errorf("unsupported public key type %T (CT only allows RSA and ECDSA)", key) + } + return nil +} + +func (key PublicKey) MarshalBinary() ([]byte, error) { + return bytes.Clone(key), nil +} + +func (key *PublicKey) UnmarshalBinary(data []byte) error { + *key = bytes.Clone(data) + return nil +} diff --git a/third_party/certspotter/ctcrypto/signatures.go b/third_party/certspotter/ctcrypto/signatures.go new file mode 100644 index 0000000..bd09095 --- /dev/null +++ b/third_party/certspotter/ctcrypto/signatures.go @@ -0,0 +1,55 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package ctcrypto + +import ( + "crypto/sha256" + "golang.org/x/crypto/cryptobyte" + + "software.sslmate.com/src/certspotter/cttypes" +) + +type SignatureInput [32]byte + +func MakeSignatureInput(message []byte) SignatureInput { + return sha256.Sum256(message) +} + +func SignatureInputForPrecertSCT(sct *cttypes.SignedCertificateTimestamp, precert cttypes.PreCert) SignatureInput { + var builder cryptobyte.Builder + builder.AddValue(sct.SCTVersion) + builder.AddValue(cttypes.CertificateTimestampSignatureType) + builder.AddUint64(sct.Timestamp) + builder.AddValue(cttypes.PrecertEntryType) + builder.AddValue(&precert) + builder.AddValue(sct.Extensions) + return MakeSignatureInput(builder.BytesOrPanic()) +} + +func SignatureInputForCertSCT(sct *cttypes.SignedCertificateTimestamp, cert cttypes.ASN1Cert) SignatureInput { + var builder cryptobyte.Builder + builder.AddValue(sct.SCTVersion) + builder.AddValue(cttypes.CertificateTimestampSignatureType) + builder.AddUint64(sct.Timestamp) + builder.AddValue(cttypes.X509EntryType) + builder.AddValue(cert) + builder.AddValue(sct.Extensions) + return MakeSignatureInput(builder.BytesOrPanic()) +} + +func SignatureInputForSTH(sth *cttypes.SignedTreeHead) SignatureInput { + var builder cryptobyte.Builder + builder.AddValue(cttypes.V1) + builder.AddValue(cttypes.TreeHashSignatureType) + builder.AddUint64(sth.Timestamp) + builder.AddUint64(sth.TreeSize) + builder.AddBytes(sth.RootHash[:]) + return MakeSignatureInput(builder.BytesOrPanic()) +} diff --git a/third_party/certspotter/cttypes/certs.go b/third_party/certspotter/cttypes/certs.go new file mode 100644 index 0000000..175df11 --- /dev/null +++ b/third_party/certspotter/cttypes/certs.go @@ -0,0 +1,121 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "fmt" + "golang.org/x/crypto/cryptobyte" +) + +type TBSCertificate []byte + +type ASN1Cert []byte + +type ASN1CertChain []ASN1Cert + +// Corresponds to the PreCert structure in RFC 6962. PreCert is a misnomer because this is really a TBSCertificate, not a precertificate. +type PreCert struct { + IssuerKeyHash [32]byte + TBSCertificate TBSCertificate +} + +type PrecertChainEntry struct { + PreCertificate ASN1Cert + PrecertificateChain ASN1CertChain +} + +func (v *TBSCertificate) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint24LengthPrefixed((*cryptobyte.String)(v)) +} +func (v TBSCertificate) Marshal(b *cryptobyte.Builder) error { + b.AddUint24LengthPrefixed(addBytesFunc(v)) + return nil +} + +func (v *ASN1Cert) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint24LengthPrefixed((*cryptobyte.String)(v)) +} +func (v ASN1Cert) Marshal(b *cryptobyte.Builder) error { + b.AddUint24LengthPrefixed(addBytesFunc(v)) + return nil +} + +func (v *ASN1CertChain) Unmarshal(s *cryptobyte.String) bool { + chainBytes := new(cryptobyte.String) + if !s.ReadUint24LengthPrefixed(chainBytes) { + return false + } + *v = []ASN1Cert{} + for !chainBytes.Empty() { + var cert ASN1Cert + if !cert.Unmarshal(chainBytes) { + return false + } + *v = append(*v, cert) + } + return true +} +func (v ASN1CertChain) Marshal(b *cryptobyte.Builder) error { + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + for _, cert := range v { + b.AddValue(cert) + } + }) + return nil +} + +func (precert *PreCert) Unmarshal(s *cryptobyte.String) error { + if !s.CopyBytes(precert.IssuerKeyHash[:]) { + return fmt.Errorf("error reading PreCert issuer_key_hash") + } + if !precert.TBSCertificate.Unmarshal(s) { + return fmt.Errorf("error reading PreCert tbs_certificate") + } + return nil +} +func (v *PreCert) Marshal(b *cryptobyte.Builder) error { + b.AddBytes(v.IssuerKeyHash[:]) + b.AddValue(v.TBSCertificate) + return nil +} + +func (entry *PrecertChainEntry) Unmarshal(s *cryptobyte.String) error { + if !entry.PreCertificate.Unmarshal(s) { + return fmt.Errorf("error reading PrecertChainEntry pre_certificate") + } + if !entry.PrecertificateChain.Unmarshal(s) { + return fmt.Errorf("error reading PrecertChainEntry preeertificate_chain") + } + return nil +} + +func ParseExtraDataForX509Entry(extraData []byte) (ASN1CertChain, error) { + str := cryptobyte.String(extraData) + var chain ASN1CertChain + if !chain.Unmarshal(&str) { + return nil, fmt.Errorf("error reading ASN.1Cert chain") + } + if !str.Empty() { + return nil, fmt.Errorf("trailing garbage after ASN.1Cert chain") + } + return chain, nil +} + +func ParseExtraDataForPrecertEntry(extraData []byte) (*PrecertChainEntry, error) { + str := cryptobyte.String(extraData) + entry := new(PrecertChainEntry) + if err := entry.Unmarshal(&str); err != nil { + return nil, err + } + if !str.Empty() { + return nil, fmt.Errorf("trailing garbage after PrecertChainEntry") + } + return entry, nil +} diff --git a/third_party/certspotter/cttypes/checkpoint.go b/third_party/certspotter/cttypes/checkpoint.go new file mode 100644 index 0000000..868457b --- /dev/null +++ b/third_party/certspotter/cttypes/checkpoint.go @@ -0,0 +1,127 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "strconv" + "strings" + + "software.sslmate.com/src/certspotter/merkletree" + "software.sslmate.com/src/certspotter/tlstypes" +) + +func chompLine(input []byte) (string, []byte, bool) { + newline := bytes.IndexByte(input, '\n') + if newline == -1 { + return "", nil, false + } + return string(input[:newline]), input[newline+1:], true +} + +func makeCheckpointKeyID(origin string, logID LogID) [4]byte { + h := sha256.New() + h.Write([]byte(origin)) + h.Write([]byte{'\n', 0x05}) + h.Write(logID[:]) + + var digest [sha256.Size]byte + h.Sum(digest[:0]) + return [4]byte(digest[:4]) +} + +func ParseCheckpoint(input []byte, logID LogID) (*SignedTreeHead, error) { + // origin + origin, input, _ := chompLine(input) + + // tree size + sizeLine, input, _ := chompLine(input) + treeSize, err := strconv.ParseUint(sizeLine, 10, 64) + if err != nil { + return nil, fmt.Errorf("malformed tree size: %w", err) + } + + // root hash + hashLine, input, _ := chompLine(input) + rootHash, err := base64.StdEncoding.DecodeString(hashLine) + if err != nil { + return nil, fmt.Errorf("malformed root hash: %w", err) + } + if len(rootHash) != merkletree.HashLen { + return nil, fmt.Errorf("root hash has wrong length (should be %d bytes long, not %d)", merkletree.HashLen, len(rootHash)) + } + + // 0 or more non-empty extension lines (ignored) + for { + line, rest, ok := chompLine(input) + if !ok { + return nil, errors.New("signed note ended prematurely") + } + input = rest + if len(line) == 0 { + break + } + } + + // signature lines + signaturePrefix := "\u2014 " + origin + " " + keyID := makeCheckpointKeyID(origin, logID) + for { + signatureLine, rest, ok := chompLine(input) + if !ok { + return nil, errors.New("signed note is missing signature from the log") + } + input = rest + if !strings.HasPrefix(signatureLine, signaturePrefix) { + continue + } + signatureBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(signatureLine, signaturePrefix)) + if err != nil { + return nil, fmt.Errorf("malformed signature: %w", err) + } + if !bytes.HasPrefix(signatureBytes, keyID[:]) { + continue + } + if len(signatureBytes) < 12 { + return nil, errors.New("malformed signature: too short") + } + timestamp := binary.BigEndian.Uint64(signatureBytes[4:12]) + signature, err := tlstypes.ParseDigitallySigned(signatureBytes[12:]) + if err != nil { + return nil, fmt.Errorf("malformed signature: %w", err) + } + return &SignedTreeHead{ + TreeSize: treeSize, + Timestamp: timestamp, + RootHash: (merkletree.Hash)(rootHash), + Signature: *signature, + }, nil + } +} + +func FormatCheckpoint(sth *SignedTreeHead, origin string, logID LogID) []byte { + keyID := makeCheckpointKeyID(origin, logID) + signature := keyID[:] + signature = binary.BigEndian.AppendUint64(signature, sth.Timestamp) + signature = append(signature, sth.Signature.Bytes()...) + + buf := new(bytes.Buffer) + fmt.Fprintln(buf, origin) + fmt.Fprintln(buf, sth.TreeSize) + fmt.Fprintln(buf, sth.RootHash.Base64String()) + fmt.Fprintln(buf) + fmt.Fprintf(buf, "\u2014 %s %s\n", origin, base64.StdEncoding.EncodeToString(signature)) + return buf.Bytes() +} diff --git a/third_party/certspotter/cttypes/checkpoint_test.go b/third_party/certspotter/cttypes/checkpoint_test.go new file mode 100644 index 0000000..1f9a286 --- /dev/null +++ b/third_party/certspotter/cttypes/checkpoint_test.go @@ -0,0 +1,113 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "bytes" + "software.sslmate.com/src/certspotter/tlstypes" + "testing" +) + +func TestParseCheckpoint(t *testing.T) { + logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f} + expected := SignedTreeHead{ + TreeSize: 820495916, + Timestamp: 1719511711300, + RootHash: [...]byte{0xc0, 0x3a, 0x6b, 0xe9, 0xf1, 0x1d, 0xc9, 0xcc, 0x20, 0x89, 0xe4, 0x45, 0xa7, 0x3f, 0x61, 0x41, 0xee, 0x82, 0xe8, 0x0d, 0x2d, 0x83, 0xa2, 0xb6, 0x65, 0x53, 0xd4, 0x96, 0xd7, 0x1e, 0xd2, 0x12}, + Signature: tlstypes.DigitallySigned{ + Algorithm: tlstypes.SignatureAndHashAlgorithm{ + Hash: tlstypes.SHA256, + Signature: tlstypes.ECDSA, + }, + Signature: []byte{0x30, 0x46, 0x02, 0x21, 0x00, 0xd0, 0x0d, 0x31, 0x91, 0x50, 0x80, 0x62, 0xfa, 0xb0, 0xf9, 0xf7, 0x63, 0x61, 0x78, 0x95, 0x2b, 0x9c, 0x19, 0x22, 0x3a, 0x1d, 0x08, 0xc5, 0x68, 0x0e, 0xd0, 0x8b, 0x3b, 0x79, 0x18, 0x88, 0x86, 0x02, 0x21, 0x00, 0xd5, 0x74, 0xae, 0x8c, 0x1d, 0x1d, 0x7a, 0x4e, 0x80, 0x6c, 0x36, 0x46, 0x81, 0xb4, 0x7c, 0x91, 0x78, 0xc0, 0x3f, 0xdc, 0xc0, 0xab, 0xa5, 0x90, 0x40, 0x8d, 0x0e, 0xf6, 0x2c, 0x83, 0xa9, 0x34}, + }, + } + + sthStrings := []string{ + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\nanother extension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n\u2014 someoneelse.example c2lnbmF0dXJlCg==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 someoneelse.example c2lnbmF0dXJlCg==\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\nanother extension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2c2lnbmF0dXJlCg==\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + } + + for i, str := range sthStrings { + parsed, err := ParseCheckpoint([]byte(str), logID) + if err != nil { + t.Errorf("%d: Unexpected error: %s", i, err) + return + } + if parsed.TreeSize != expected.TreeSize { + t.Errorf("%d: wrong tree size", i) + return + } + if parsed.Timestamp != expected.Timestamp { + t.Errorf("%d: wrong timestamp", i) + return + } + if !bytes.Equal(parsed.RootHash[:], expected.RootHash[:]) { + t.Errorf("%d: wrong root hash", i) + return + } + if !(parsed.Signature.Algorithm == expected.Signature.Algorithm && bytes.Equal(parsed.Signature.Signature, expected.Signature.Signature)) { + t.Errorf("%d: wrong signature", i) + return + } + } +} + +func TestParseCheckpointFailure(t *testing.T) { + logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f} + sthStrings := []string{ + "", + "\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 oak.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 bm90YXNpZ25hdHVyZQo=\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 notbase64!\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916!\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEd!ycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n- sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n", + } + + for i, str := range sthStrings { + _, err := ParseCheckpoint([]byte(str), logID) + if err == nil { + t.Errorf("%d: Unexpected success", i) + return + } + } +} + +func TestFormatCheckpoint(t *testing.T) { + origin := "sycamore.ct.letsencrypt.org/2024h2" + logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f} + sth := SignedTreeHead{ + TreeSize: 820495916, + Timestamp: 1719511711300, + RootHash: [...]byte{0xc0, 0x3a, 0x6b, 0xe9, 0xf1, 0x1d, 0xc9, 0xcc, 0x20, 0x89, 0xe4, 0x45, 0xa7, 0x3f, 0x61, 0x41, 0xee, 0x82, 0xe8, 0x0d, 0x2d, 0x83, 0xa2, 0xb6, 0x65, 0x53, 0xd4, 0x96, 0xd7, 0x1e, 0xd2, 0x12}, + Signature: tlstypes.DigitallySigned{ + Algorithm: tlstypes.SignatureAndHashAlgorithm{ + Hash: tlstypes.SHA256, + Signature: tlstypes.ECDSA, + }, + Signature: []byte{0x30, 0x46, 0x02, 0x21, 0x00, 0xd0, 0x0d, 0x31, 0x91, 0x50, 0x80, 0x62, 0xfa, 0xb0, 0xf9, 0xf7, 0x63, 0x61, 0x78, 0x95, 0x2b, 0x9c, 0x19, 0x22, 0x3a, 0x1d, 0x08, 0xc5, 0x68, 0x0e, 0xd0, 0x8b, 0x3b, 0x79, 0x18, 0x88, 0x86, 0x02, 0x21, 0x00, 0xd5, 0x74, 0xae, 0x8c, 0x1d, 0x1d, 0x7a, 0x4e, 0x80, 0x6c, 0x36, 0x46, 0x81, 0xb4, 0x7c, 0x91, 0x78, 0xc0, 0x3f, 0xdc, 0xc0, 0xab, 0xa5, 0x90, 0x40, 0x8d, 0x0e, 0xf6, 0x2c, 0x83, 0xa9, 0x34}, + }, + } + + expected := "sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n" + + got := FormatCheckpoint(&sth, origin, logID) + if string(got) != expected { + t.Errorf("expected %q, got %q", expected, got) + } +} diff --git a/third_party/certspotter/cttypes/helpers.go b/third_party/certspotter/cttypes/helpers.go new file mode 100644 index 0000000..9546ad8 --- /dev/null +++ b/third_party/certspotter/cttypes/helpers.go @@ -0,0 +1,20 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "golang.org/x/crypto/cryptobyte" +) + +func addBytesFunc(v []byte) cryptobyte.BuilderContinuation { + return func(b *cryptobyte.Builder) { + b.AddBytes(v) + } +} diff --git a/third_party/certspotter/cttypes/logid.go b/third_party/certspotter/cttypes/logid.go new file mode 100644 index 0000000..9e44aaa --- /dev/null +++ b/third_party/certspotter/cttypes/logid.go @@ -0,0 +1,69 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "bytes" + "encoding/base64" + "fmt" + "golang.org/x/crypto/cryptobyte" +) + +type LogID [32]byte + +func (id LogID) Compare(other LogID) int { + return bytes.Compare(id[:], other[:]) +} + +func (v *LogID) Unmarshal(s *cryptobyte.String) bool { + return s.CopyBytes((*v)[:]) +} +func (v LogID) Marshal(b *cryptobyte.Builder) error { + b.AddBytes(v[:]) + return nil +} + +func (id *LogID) UnmarshalBinary(bytes []byte) error { + if len(bytes) != len(*id) { + return fmt.Errorf("LogID has wrong length (should be %d, not %d)", len(*id), len(bytes)) + } + *id = (LogID)(bytes) + return nil +} +func (id LogID) MarshalBinary() ([]byte, error) { + return id[:], nil +} + +func (id *LogID) UnmarshalText(textData []byte) error { + if len(textData) != 44 { + return fmt.Errorf("LogID has wrong length (should be %d, not %d)", 44, len(textData)) + } + var bytes [33]byte + if n, err := base64.StdEncoding.Decode(bytes[:], textData); err != nil { + return fmt.Errorf("LogID contains invalid base64: %w", err) + } else if n != 32 { + return fmt.Errorf("LogID has wrong length (should be %d bytes, not %d)", 32, n) + } + copy(id[:], bytes[:]) + return nil +} +func (id LogID) MarshalText() ([]byte, error) { + encodedBytes := make([]byte, 44) + base64.StdEncoding.Encode(encodedBytes, id[:]) + return encodedBytes, nil +} + +func (id LogID) Base64String() string { + return base64.StdEncoding.EncodeToString(id[:]) +} + +func (id LogID) Base64URLString() string { + return base64.RawURLEncoding.EncodeToString(id[:]) +} diff --git a/third_party/certspotter/cttypes/merkleleaf.go b/third_party/certspotter/cttypes/merkleleaf.go new file mode 100644 index 0000000..2dc6190 --- /dev/null +++ b/third_party/certspotter/cttypes/merkleleaf.go @@ -0,0 +1,213 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "fmt" + "golang.org/x/crypto/cryptobyte" + "software.sslmate.com/src/certspotter/merkletree" +) + +type MerkleLeafType uint8 + +const ( + TimestampedEntryType MerkleLeafType = 0 +) + +type LogEntryType uint16 + +const ( + X509EntryType LogEntryType = 0 + PrecertEntryType LogEntryType = 1 +) + +type CTExtensions []byte + +type MerkleTreeLeaf struct { + Version Version + LeafType MerkleLeafType + TimestampedEntry *TimestampedEntry +} + +type TimestampedEntry struct { + Timestamp uint64 + EntryType LogEntryType + SignedEntryASN1Cert *ASN1Cert + SignedEntryPreCert *PreCert + Extensions CTExtensions +} + +func (v *MerkleLeafType) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint8((*uint8)(v)) +} +func (v MerkleLeafType) Marshal(b *cryptobyte.Builder) error { + b.AddUint8(uint8(v)) + return nil +} + +func (v *LogEntryType) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint16((*uint16)(v)) +} +func (v LogEntryType) Marshal(b *cryptobyte.Builder) error { + b.AddUint16(uint16(v)) + return nil +} + +func (v *CTExtensions) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint16LengthPrefixed((*cryptobyte.String)(v)) +} +func (v CTExtensions) Marshal(b *cryptobyte.Builder) error { + b.AddUint16LengthPrefixed(addBytesFunc(v)) + return nil +} + +func (leaf *MerkleTreeLeaf) Unmarshal(s *cryptobyte.String) error { + if !leaf.Version.Unmarshal(s) { + return fmt.Errorf("error reading MerkleTreeLeaf version") + } + if leaf.Version != V1 { + return fmt.Errorf("unsupported Version 0x%02x", leaf.Version) + } + if !leaf.LeafType.Unmarshal(s) { + return fmt.Errorf("error reading MerkleTreeLeaf leaf_type") + } + switch leaf.LeafType { + case TimestampedEntryType: + leaf.TimestampedEntry = new(TimestampedEntry) + if err := leaf.TimestampedEntry.Unmarshal(s); err != nil { + return err + } + default: + return fmt.Errorf("unrecognized MerkleLeafType 0x%02x", leaf.LeafType) + } + return nil +} +func (v *MerkleTreeLeaf) Marshal(b *cryptobyte.Builder) error { + b.AddValue(v.Version) + b.AddValue(v.LeafType) + switch v.LeafType { + case TimestampedEntryType: + b.AddValue(v.TimestampedEntry) + } + return nil +} +func (v *MerkleTreeLeaf) Bytes() ([]byte, error) { + var builder cryptobyte.Builder + builder.AddValue(v) + return builder.Bytes() +} +func (v *MerkleTreeLeaf) Hash() merkletree.Hash { + var builder cryptobyte.Builder + builder.AddValue(v) + return merkletree.HashLeaf(builder.BytesOrPanic()) +} + +func (entry *TimestampedEntry) Unmarshal(s *cryptobyte.String) error { + if !s.ReadUint64(&entry.Timestamp) { + return fmt.Errorf("error reading TimestampedEntry timestamp") + } + if !entry.EntryType.Unmarshal(s) { + return fmt.Errorf("error reading TimestampedEntry entry_type") + } + switch entry.EntryType { + case X509EntryType: + entry.SignedEntryASN1Cert = new(ASN1Cert) + if !entry.SignedEntryASN1Cert.Unmarshal(s) { + return fmt.Errorf("error reading TimestampedEntry signed_entry ASN.1Cert") + } + case PrecertEntryType: + entry.SignedEntryPreCert = new(PreCert) + if err := entry.SignedEntryPreCert.Unmarshal(s); err != nil { + return fmt.Errorf("error reading TimestampedEntryType signed_entry: %w", err) + } + default: + return fmt.Errorf("unrecognized TimestampedEntryType 0x%02x", entry.EntryType) + } + if !entry.Extensions.Unmarshal(s) { + return fmt.Errorf("error reading TimestampedEntry extensions") + } + return nil +} +func (v *TimestampedEntry) Marshal(b *cryptobyte.Builder) error { + b.AddUint64(v.Timestamp) + b.AddValue(v.EntryType) + switch v.EntryType { + case X509EntryType: + b.AddValue(v.SignedEntryASN1Cert) + case PrecertEntryType: + b.AddValue(v.SignedEntryPreCert) + } + b.AddValue(v.Extensions) + return nil +} + +func ParseLeafInput(leafInput []byte) (*MerkleTreeLeaf, error) { + str := cryptobyte.String(leafInput) + leaf := new(MerkleTreeLeaf) + if err := leaf.Unmarshal(&str); err != nil { + return nil, err + } + if !str.Empty() { + return nil, fmt.Errorf("trailing garbage after MerkleTreeLeaf") + } + return leaf, nil +} + +func MerkleTreeLeafForCert(timestamp uint64, extensions []byte, cert ASN1Cert) *MerkleTreeLeaf { + return &MerkleTreeLeaf{ + Version: V1, + LeafType: TimestampedEntryType, + TimestampedEntry: &TimestampedEntry{ + Timestamp: timestamp, + EntryType: X509EntryType, + SignedEntryASN1Cert: &cert, + Extensions: extensions, + }, + } +} + +func MerkleTreeLeafForCertSCT(sct *SignedCertificateTimestamp, cert ASN1Cert) *MerkleTreeLeaf { + return &MerkleTreeLeaf{ + Version: sct.SCTVersion, + LeafType: TimestampedEntryType, + TimestampedEntry: &TimestampedEntry{ + Timestamp: sct.Timestamp, + EntryType: X509EntryType, + SignedEntryASN1Cert: &cert, + Extensions: sct.Extensions, + }, + } +} + +func MerkleTreeLeafForPrecert(timestamp uint64, extensions []byte, precert PreCert) *MerkleTreeLeaf { + return &MerkleTreeLeaf{ + Version: V1, + LeafType: TimestampedEntryType, + TimestampedEntry: &TimestampedEntry{ + Timestamp: timestamp, + EntryType: PrecertEntryType, + SignedEntryPreCert: &precert, + Extensions: extensions, + }, + } +} + +func MerkleTreeLeafForPrecertSCT(sct *SignedCertificateTimestamp, precert PreCert) *MerkleTreeLeaf { + return &MerkleTreeLeaf{ + Version: sct.SCTVersion, + LeafType: TimestampedEntryType, + TimestampedEntry: &TimestampedEntry{ + Timestamp: sct.Timestamp, + EntryType: PrecertEntryType, + SignedEntryPreCert: &precert, + Extensions: sct.Extensions, + }, + } +} diff --git a/third_party/certspotter/cttypes/sct.go b/third_party/certspotter/cttypes/sct.go new file mode 100644 index 0000000..f11dcb1 --- /dev/null +++ b/third_party/certspotter/cttypes/sct.go @@ -0,0 +1,73 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "fmt" + "golang.org/x/crypto/cryptobyte" + "software.sslmate.com/src/certspotter/tlstypes" +) + +type SignedCertificateTimestamp struct { + SCTVersion Version `json:"sct_version"` + ID LogID `json:"id"` + Timestamp uint64 `json:"timestamp"` + Extensions CTExtensions `json:"extensions"` + Signature tlstypes.DigitallySigned `json:"signature"` +} + +func (sct *SignedCertificateTimestamp) Bytes() ([]byte, error) { + var builder cryptobyte.Builder + builder.AddValue(sct) + return builder.Bytes() +} + +func (sct *SignedCertificateTimestamp) Marshal(b *cryptobyte.Builder) error { + b.AddValue(sct.SCTVersion) + b.AddValue(sct.ID) + b.AddUint64(sct.Timestamp) + b.AddValue(sct.Extensions) + b.AddValue(sct.Signature) + return nil +} + +func (sct *SignedCertificateTimestamp) Unmarshal(s *cryptobyte.String) error { + if !sct.SCTVersion.Unmarshal(s) { + return fmt.Errorf("error reading SCT version") + } + if sct.SCTVersion != V1 { + return fmt.Errorf("unsupported SCT version 0x%02x", sct.SCTVersion) + } + if !sct.ID.Unmarshal(s) { + return fmt.Errorf("error reading SCT id") + } + if !s.ReadUint64(&sct.Timestamp) { + return fmt.Errorf("error reading SCT timestamp") + } + if !sct.Extensions.Unmarshal(s) { + return fmt.Errorf("error reading SCT extensions") + } + if !sct.Signature.Unmarshal(s) { + return fmt.Errorf("error reading SCT signature") + } + return nil +} + +func ParseSignedCertificateTimestamp(data []byte) (*SignedCertificateTimestamp, error) { + str := cryptobyte.String(data) + sct := new(SignedCertificateTimestamp) + if err := sct.Unmarshal(&str); err != nil { + return nil, err + } + if !str.Empty() { + return nil, fmt.Errorf("trailing garbage after SignedCertificateTimestamp") + } + return sct, nil +} diff --git a/third_party/certspotter/cttypes/signatures.go b/third_party/certspotter/cttypes/signatures.go new file mode 100644 index 0000000..ac59536 --- /dev/null +++ b/third_party/certspotter/cttypes/signatures.go @@ -0,0 +1,29 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "golang.org/x/crypto/cryptobyte" +) + +type SignatureType uint8 + +const ( + CertificateTimestampSignatureType SignatureType = 0 + TreeHashSignatureType SignatureType = 1 +) + +func (v *SignatureType) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint8((*uint8)(v)) +} +func (v SignatureType) Marshal(b *cryptobyte.Builder) error { + b.AddUint8(uint8(v)) + return nil +} diff --git a/third_party/certspotter/cttypes/sth.go b/third_party/certspotter/cttypes/sth.go new file mode 100644 index 0000000..aec9d2a --- /dev/null +++ b/third_party/certspotter/cttypes/sth.go @@ -0,0 +1,37 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "software.sslmate.com/src/certspotter/merkletree" + "software.sslmate.com/src/certspotter/tlstypes" + "time" +) + +type SignedTreeHead struct { + TreeSize uint64 `json:"tree_size"` + Timestamp uint64 `json:"timestamp"` + RootHash merkletree.Hash `json:"sha256_root_hash"` + Signature tlstypes.DigitallySigned `json:"tree_head_signature"` +} + +type GossipedSignedTreeHead struct { + SignedTreeHead + STHVersion Version `json:"sth_version"` + LogID LogID `json:"log_id"` +} + +func (sth *SignedTreeHead) TimestampTime() time.Time { + return time.UnixMilli(int64(sth.Timestamp)) +} + +func (sth *SignedTreeHead) Same(other *SignedTreeHead) bool { + return sth.TreeSize == other.TreeSize && sth.Timestamp == other.Timestamp && sth.RootHash == other.RootHash +} diff --git a/third_party/certspotter/cttypes/version.go b/third_party/certspotter/cttypes/version.go new file mode 100644 index 0000000..b35b574 --- /dev/null +++ b/third_party/certspotter/cttypes/version.go @@ -0,0 +1,29 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package cttypes + +import ( + "golang.org/x/crypto/cryptobyte" +) + +type Version uint8 + +const ( + V1 Version = 0 +) + +func (v Version) Marshal(b *cryptobyte.Builder) error { + b.AddUint8(uint8(v)) + return nil +} + +func (v *Version) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint8((*uint8)(v)) +} diff --git a/third_party/certspotter/go.mod b/third_party/certspotter/go.mod new file mode 100644 index 0000000..4db02f7 --- /dev/null +++ b/third_party/certspotter/go.mod @@ -0,0 +1,14 @@ +module software.sslmate.com/src/certspotter + +go 1.25.0 + +require ( + golang.org/x/crypto v0.52.0 + golang.org/x/net v0.55.0 + golang.org/x/sync v0.20.0 + golang.org/x/time v0.15.0 +) + +require golang.org/x/text v0.37.0 // indirect + +retract v0.19.0 // Contains serious bugs. diff --git a/third_party/certspotter/go.sum b/third_party/certspotter/go.sum new file mode 100644 index 0000000..9368646 --- /dev/null +++ b/third_party/certspotter/go.sum @@ -0,0 +1,22 @@ +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/third_party/certspotter/helpers.go b/third_party/certspotter/helpers.go new file mode 100644 index 0000000..56c75f7 --- /dev/null +++ b/third_party/certspotter/helpers.go @@ -0,0 +1,80 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "math/big" +) + +type CertInfo struct { + TBS *TBSCertificate + + Subject RDNSequence + SubjectParseError error + Issuer RDNSequence + IssuerParseError error + SANs []SubjectAltName + SANsParseError error + SerialNumber *big.Int + SerialNumberParseError error + Validity *CertValidity + ValidityParseError error + IsCA *bool + IsCAParseError error + IsPreCert bool +} + +func MakeCertInfoFromTBS(tbs *TBSCertificate) *CertInfo { + info := &CertInfo{TBS: tbs} + + info.Subject, info.SubjectParseError = tbs.ParseSubject() + info.Issuer, info.IssuerParseError = tbs.ParseIssuer() + info.SANs, info.SANsParseError = tbs.ParseSubjectAltNames() + info.SerialNumber, info.SerialNumberParseError = tbs.ParseSerialNumber() + info.Validity, info.ValidityParseError = tbs.ParseValidity() + info.IsCA, info.IsCAParseError = tbs.ParseBasicConstraints() + info.IsPreCert = len(tbs.GetExtension(oidExtensionCTPoison)) > 0 + + return info +} + +func MakeCertInfoFromRawTBS(tbsBytes []byte) (*CertInfo, error) { + tbs, err := ParseTBSCertificate(tbsBytes) + if err != nil { + return nil, err + } + return MakeCertInfoFromTBS(tbs), nil +} + +func MakeCertInfoFromRawCert(certBytes []byte) (*CertInfo, error) { + cert, err := ParseCertificate(certBytes) + if err != nil { + return nil, err + } + return MakeCertInfoFromRawTBS(cert.GetRawTBSCertificate()) +} + +func MatchesWildcard(dnsName string, pattern string) bool { + for len(pattern) > 0 { + if pattern[0] == '*' { + if len(dnsName) > 0 && dnsName[0] != '.' && MatchesWildcard(dnsName[1:], pattern) { + return true + } + pattern = pattern[1:] + } else { + if len(dnsName) == 0 || pattern[0] != dnsName[0] { + return false + } + pattern = pattern[1:] + dnsName = dnsName[1:] + } + } + return len(dnsName) == 0 +} diff --git a/third_party/certspotter/helpers_test.go b/third_party/certspotter/helpers_test.go new file mode 100644 index 0000000..5b1229c --- /dev/null +++ b/third_party/certspotter/helpers_test.go @@ -0,0 +1,49 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "testing" +) + +func doWildcardTest(t *testing.T, dnsName string, wildcard string, expected bool) { + if MatchesWildcard(dnsName, wildcard) != expected { + t.Errorf("MatchesWildcard(%q, %q) != %v", dnsName, wildcard, expected) + } +} + +func TestMatchesWildcard(t *testing.T) { + doWildcardTest(t, "", "", true) + doWildcardTest(t, "example.com", "example.com", true) + doWildcardTest(t, "example.org", "example.com", false) + doWildcardTest(t, "example.com", "", false) + doWildcardTest(t, "", "example.com", false) + doWildcardTest(t, "", "*.example.com", false) + doWildcardTest(t, "", "exam*ple.com", false) + doWildcardTest(t, "", "exam*ple.co*m", false) + doWildcardTest(t, "example.org", "example.com", false) + doWildcardTest(t, "example.org", "*.example.com", false) + doWildcardTest(t, "example.org", "exam*ple.com", false) + doWildcardTest(t, "example.org", "exam*ple.co*m", false) + doWildcardTest(t, "example.com", "*.example.com", false) + doWildcardTest(t, "www.example.com", "*.example.com", true) + doWildcardTest(t, "", "*", true) + doWildcardTest(t, "", "****", true) + doWildcardTest(t, "a", "****", true) + doWildcardTest(t, "a", "*", true) + doWildcardTest(t, "a", "****", true) + doWildcardTest(t, "abcd", "****", true) + doWildcardTest(t, "abcdef", "****", true) + doWildcardTest(t, "www-example.com", "*-example.com", true) + doWildcardTest(t, "www-example-www.com", "*-example-*.com", true) + doWildcardTest(t, "examplecom", "example*", true) + doWildcardTest(t, "example.com", "example*", false) + doWildcardTest(t, "examplea.com", "example*", false) +} diff --git a/third_party/certspotter/identifiers.go b/third_party/certspotter/identifiers.go new file mode 100644 index 0000000..dd73a45 --- /dev/null +++ b/third_party/certspotter/identifiers.go @@ -0,0 +1,301 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "bytes" + "golang.org/x/net/idna" + "net" + "strings" + "unicode/utf8" +) + +const UnparsableDNSLabelPlaceholder = "" + +/* +const ( + IdentifierSourceSubjectCN = iota + IdentifierSourceDNSName + IdentifierSourceIPAddr +) +type IdentifierSource int + +type UnknownIdentifier struct { + Source IdentifierSource + Value []byte +} +*/ + +type Identifiers struct { + DNSNames []string // stored as ASCII, with IDNs in Punycode + IPAddrs []net.IP + //Unknowns []UnknownIdentifier +} + +func NewIdentifiers() *Identifiers { + return &Identifiers{ + DNSNames: []string{}, + IPAddrs: []net.IP{}, + //Unknowns: []UnknownIdentifier{}, + } +} + +func parseIPAddrString(str string) net.IP { + return net.ParseIP(str) +} + +func isASCIIString(value []byte) bool { + for _, b := range value { + if b > 127 { + return false + } + } + return true +} +func isUTF8String(value []byte) bool { + return utf8.Valid(value) +} +func latin1ToUTF8(value []byte) string { + runes := make([]rune, len(value)) + for i, b := range value { + runes[i] = rune(b) + } + return string(runes) +} + +// Make sure the DNS label doesn't have any weird characters that +// could cause trouble during later processing. +func isSaneDNSLabelChar(ch rune) bool { + return ch == '\t' || (ch >= 32 && ch <= 126) +} +func isSaneDNSLabel(label string) bool { + for _, ch := range label { + if !isSaneDNSLabelChar(ch) { + return false + } + } + return true +} + +func trimHttpPrefixString(value string) string { + if strings.HasPrefix(value, "http://") { + return value[7:] + } else if strings.HasPrefix(value, "https://") { + return value[8:] + } else { + return value + } +} + +func trimHttpPrefixBytes(value []byte) []byte { + if bytes.HasPrefix(value, []byte("http://")) { + return value[7:] + } else if bytes.HasPrefix(value, []byte("https://")) { + return value[8:] + } else { + return value + } +} + +func trimTrailingDots(value string) string { + length := len(value) + for length > 0 && value[length-1] == '.' { + length-- + } + return value[0:length] +} + +// Try to canonicalize/sanitize the DNS name: +// 1. Trim leading and trailing whitespace +// 2. Trim trailing dots +// 3. Convert to lower case +// 4. Replace totally nonsensical labels (e.g. having non-printable characters) with a placeholder +func sanitizeDNSName(value string) string { + value = strings.ToLower(trimTrailingDots(strings.TrimSpace(value))) + labels := strings.Split(value, ".") + for i, label := range labels { + if !isSaneDNSLabel(label) { + labels[i] = UnparsableDNSLabelPlaceholder + } + } + return strings.Join(labels, ".") +} + +// Like sanitizeDNSName, but labels that are Unicode are converted to Punycode. +func sanitizeUnicodeDNSName(value string) string { + value = strings.ToLower(trimTrailingDots(strings.TrimSpace(value))) + labels := strings.Split(value, ".") + for i, label := range labels { + if asciiLabel, err := idna.ToASCII(label); err == nil && isSaneDNSLabel(asciiLabel) { + labels[i] = asciiLabel + } else { + labels[i] = UnparsableDNSLabelPlaceholder + } + } + return strings.Join(labels, ".") +} + +func (ids *Identifiers) appendDNSName(dnsName string) { + if dnsName != "" && !ids.hasDNSName(dnsName) { + ids.DNSNames = append(ids.DNSNames, dnsName) + } +} +func (ids *Identifiers) appendIPAddress(ipaddr net.IP) { + if !ids.hasIPAddress(ipaddr) { + ids.IPAddrs = append(ids.IPAddrs, ipaddr) + } +} + +func (ids *Identifiers) hasDNSName(target string) bool { + for _, value := range ids.DNSNames { + if value == target { + return true + } + } + return false +} +func (ids *Identifiers) hasIPAddress(target net.IP) bool { + for _, value := range ids.IPAddrs { + if value.Equal(target) { + return true + } + } + return false +} + +func (ids *Identifiers) addDnsSANfinal(value []byte) { + if ipaddr := parseIPAddrString(string(value)); ipaddr != nil { + // Stupid CAs put IP addresses in DNS SANs because stupid Microsoft + // used to not support IP address SANs. Since there's no way for an IP + // address to also be a valid DNS name, just treat it like an IP address + // and not try to process it as a DNS name. + ids.appendIPAddress(ipaddr) + } else if isASCIIString(value) { + ids.appendDNSName(sanitizeDNSName(string(value))) + } else { + // DNS SANs are supposed to be IA5Strings (i.e. ASCII) but CAs can't follow + // simple rules. Unfortunately, we have no idea what the encoding really is + // in this case, so interpret it as both UTF-8 (if it's valid UTF-8) + // and Latin-1. + if isUTF8String(value) { + ids.appendDNSName(sanitizeUnicodeDNSName(string(value))) + } + ids.appendDNSName(sanitizeUnicodeDNSName(latin1ToUTF8(value))) + } +} + +func (ids *Identifiers) addDnsSANnonull(value []byte) { + if slashIndex := bytes.IndexByte(value, '/'); slashIndex != -1 { + // If the value contains a slash, then this might be a URL, + // so process the part of the value up to the first slash, + // which should be the domain. Even though no client should + // ever successfully validate such a DNS name, the domain owner + // might still want to know about it. + ids.addDnsSANfinal(value[0:slashIndex]) + } + ids.addDnsSANfinal(value) +} + +func (ids *Identifiers) AddDnsSAN(value []byte) { + // Trim http:// and https:// prefixes, which are all too common in the wild, + // so http://example.com becomes just example.com. Even though clients + // should never successfully validate a DNS name like http://example.com, + // the owner of example.com might still want to know about it. + value = trimHttpPrefixBytes(value) + + if nullIndex := bytes.IndexByte(value, 0); nullIndex != -1 { + // If the value contains a null byte, process the part of + // the value up to the first null byte in addition to the + // complete value, in case this certificate is an attempt to + // fake out validators that only compare up to the first null. + ids.addDnsSANnonull(value[0:nullIndex]) + } + ids.addDnsSANnonull(value) +} + +func (ids *Identifiers) addCNfinal(value string) { + if ipaddr := parseIPAddrString(value); ipaddr != nil { + ids.appendIPAddress(ipaddr) + } else if !strings.ContainsRune(value, ' ') { + // If the CN contains a space it's clearly not a DNS name, so ignore it. + ids.appendDNSName(sanitizeUnicodeDNSName(value)) + } +} + +func (ids *Identifiers) addCNnonull(value string) { + if slashIndex := strings.IndexRune(value, '/'); slashIndex != -1 { + // If the value contains a slash, then this might be a URL, + // so process the part of the value up to the first slash, + // which should be the domain. Even though no client should + // ever successfully validate such a DNS name, the domain owner + // might still want to know about it. + ids.addCNfinal(value[0:slashIndex]) + } + ids.addCNfinal(value) +} + +func (ids *Identifiers) AddCN(value string) { + // Trim http:// and https:// prefixes, which are all too common in the wild, + // so http://example.com becomes just example.com. Even though clients + // should never successfully validate a DNS name like http://example.com, + // the owner of example.com might still want to know about it. + value = trimHttpPrefixString(value) + + if nullIndex := strings.IndexRune(value, 0); nullIndex != -1 { + // If the value contains a null byte, process the part of + // the value up to the first null byte in addition to the + // complete value, in case this certificate is an attempt to + // fake out validators that only compare up to the first null. + ids.addCNnonull(value[0:nullIndex]) + } + ids.addCNnonull(value) +} + +func (ids *Identifiers) AddIPAddress(value net.IP) { + ids.appendIPAddress(value) +} + +func (cert *CertInfo) ParseIdentifiers() (*Identifiers, error) { + ids := NewIdentifiers() + + if cert.SubjectParseError != nil { + return nil, cert.SubjectParseError + } + cns, err := cert.Subject.ParseCNs() + if err != nil { + return nil, err + } + for _, cn := range cns { + ids.AddCN(cn) + } + + if cert.SANsParseError != nil { + return nil, cert.SANsParseError + } + for _, san := range cert.SANs { + switch san.Type { + case sanDNSName: + ids.AddDnsSAN(san.Value) + case sanIPAddress: + if len(san.Value) == 4 || len(san.Value) == 16 { + ids.AddIPAddress(net.IP(san.Value)) + } + // TODO: decide what to do with IP addresses with an invalid length. + // The two encoding errors I've observed in CT logs are: + // 1. encoding the IP address as a string + // 2. a value of 0x00000000FFFFFF00 (WTF?) + // IP addresses aren't a high priority so just ignore invalid ones for now. + // Hopefully no clients out there are dumb enough to process IP address + // SANs encoded as strings... + } + } + + return ids, nil +} diff --git a/third_party/certspotter/loglist/helpers.go b/third_party/certspotter/loglist/helpers.go new file mode 100644 index 0000000..ab1586b --- /dev/null +++ b/third_party/certspotter/loglist/helpers.go @@ -0,0 +1,40 @@ +// Copyright (C) 2020 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package loglist + +import ( + "time" +) + +// Return all tiled and non-tiled logs from all operators +func (list *List) AllLogs() []*Log { + logs := []*Log{} + for operator := range list.Operators { + for log := range list.Operators[operator].Logs { + logs = append(logs, &list.Operators[operator].Logs[log]) + } + for log := range list.Operators[operator].TiledLogs { + logs = append(logs, &list.Operators[operator].TiledLogs[log]) + } + } + return logs +} + +func (log *Log) LogIDString() string { + return log.LogID.Base64String() +} + +func (log *Log) AcceptsExpiration(expiration time.Time) bool { + return log.TemporalInterval == nil || withinInterval(expiration, log.TemporalInterval.StartInclusive, log.TemporalInterval.EndExclusive) +} + +func withinInterval(expiration, startInclusive, endExclusive time.Time) bool { + return !expiration.Before(startInclusive) && expiration.Before(endExclusive) +} diff --git a/third_party/certspotter/loglist/load.go b/third_party/certspotter/loglist/load.go new file mode 100644 index 0000000..ee85b5b --- /dev/null +++ b/third_party/certspotter/loglist/load.go @@ -0,0 +1,129 @@ +// Copyright (C) 2020, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package loglist + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +var UserAgent = "" + +// maxResponseBytes bounds the size of a log list we are willing to read into memory. +const maxResponseBytes = 16 << 20 // 16 MiB + +// httpClient is used for fetching log lists. +var httpClient = &http.Client{ + Timeout: 60 * time.Second, +} + +type ModificationToken struct { + etag string + modified time.Time +} + +var ErrNotModified = errors.New("loglist has not been modified") + +func newModificationToken(response *http.Response) *ModificationToken { + token := &ModificationToken{ + etag: response.Header.Get("ETag"), + } + if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil { + token.modified = t + } + return token +} + +func (token *ModificationToken) setRequestHeaders(request *http.Request) { + if token.etag != "" { + request.Header.Set("If-None-Match", token.etag) + } else if !token.modified.IsZero() { + request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat)) + } +} + +func Load(ctx context.Context, urlOrFile string) (*List, error) { + list, _, err := LoadIfModified(ctx, urlOrFile, nil) + return list, err +} + +func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) { + if strings.HasPrefix(urlOrFile, "https://") { + return FetchIfModified(ctx, urlOrFile, token) + } else { + list, err := ReadFile(urlOrFile) + return list, nil, err + } +} + +func Fetch(ctx context.Context, url string) (*List, error) { + list, _, err := FetchIfModified(ctx, url, nil) + return list, err +} + +func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, nil, err + } + request.Header.Set("User-Agent", UserAgent) + if token != nil { + token.setRequestHeaders(request) + } + response, err := httpClient.Do(request) + if err != nil { + return nil, nil, err + } + content, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1)) + response.Body.Close() + if err != nil { + return nil, nil, err + } + if len(content) > maxResponseBytes { + return nil, nil, fmt.Errorf("%s: response body exceeds maximum allowed size of %d bytes", url, maxResponseBytes) + } + if token != nil && response.StatusCode == http.StatusNotModified { + return nil, nil, ErrNotModified + } + if response.StatusCode != 200 { + return nil, nil, fmt.Errorf("%s: %s", url, response.Status) + } + list, err := Unmarshal(content) + if err != nil { + return nil, nil, fmt.Errorf("error parsing %s: %w", url, err) + } + return list, newModificationToken(response), err +} + +func ReadFile(filename string) (*List, error) { + content, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return Unmarshal(content) +} + +func Unmarshal(jsonBytes []byte) (*List, error) { + list := new(List) + if err := json.Unmarshal(jsonBytes, list); err != nil { + return nil, err + } + if err := list.Validate(); err != nil { + return nil, fmt.Errorf("invalid log list: %s", err) + } + return list, nil +} diff --git a/third_party/certspotter/loglist/mozilla/ctlogs.go b/third_party/certspotter/loglist/mozilla/ctlogs.go new file mode 100644 index 0000000..1236386 --- /dev/null +++ b/third_party/certspotter/loglist/mozilla/ctlogs.go @@ -0,0 +1,275 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +// Package mozilla contains a parser for Mozilla's CTKnownLogs.h file +package mozilla + +import ( + "bufio" + "errors" + "io" + "strconv" + "strings" + "time" +) + +// CTLogInfo describes a certificate transparency log from Mozilla's CTKnownLogs.h +// file. +type CTLogInfo struct { + Name string + State string // Admissible or Retired + Protocol string // RFC6962 or Tiled + Timestamp time.Time + OperatorIndex int + Key []byte +} + +// CTLogOperatorInfo describes a CT log operator from Mozilla's CTKnownLogs.h +// file. +type CTLogOperatorInfo struct { + Name string + ID int +} + +// Parse reads the CTKnownLogs.h content from r and returns the parsed logs and +// operators. Blocks enclosed by `#ifdef DEBUG` and `#endif` are ignored. +func Parse(r io.Reader) ([]CTLogInfo, []CTLogOperatorInfo, error) { + scanner := bufio.NewScanner(r) + skip := 0 + inLogs := false + inOps := false + + var logs []CTLogInfo + var ops []CTLogOperatorInfo + + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + + switch { + case strings.HasPrefix(trimmed, "#ifdef DEBUG"): + skip++ + continue + case strings.HasPrefix(trimmed, "#endif"): + if skip > 0 { + skip-- + continue + } + } + + if skip > 0 { + continue + } + + if strings.HasPrefix(trimmed, "const CTLogInfo kCTLogList[]") { + inLogs = true + continue + } + if strings.HasPrefix(trimmed, "const CTLogOperatorInfo kCTLogOperatorList[]") { + inOps = true + continue + } + + if inLogs { + if trimmed == "};" { + inLogs = false + continue + } + if strings.HasPrefix(trimmed, "{") { + log, err := readLogEntry(trimmed, scanner) + if err != nil { + return nil, nil, err + } + logs = append(logs, log) + } + continue + } + + if inOps { + if trimmed == "};" { + inOps = false + continue + } + if strings.HasPrefix(trimmed, "{") { + op, err := readOperatorEntry(trimmed) + if err != nil { + return nil, nil, err + } + ops = append(ops, op) + } + } + } + + if err := scanner.Err(); err != nil { + return nil, nil, err + } + + return logs, ops, nil +} + +func readLogEntry(firstLine string, s *bufio.Scanner) (CTLogInfo, error) { + var log CTLogInfo + + // Example first line: + // {"Name", CTLogState::Admissible, CTLogFormat::RFC6962, + firstLine = strings.TrimSpace(firstLine) + if !strings.HasPrefix(firstLine, "{") { + return log, errors.New("invalid log entry start") + } + firstLine = strings.TrimPrefix(firstLine, "{") + firstLine = strings.TrimSuffix(firstLine, ",") + + parts := splitCSV(firstLine) + + for len(parts) < 3 { + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + next := strings.TrimSpace(strings.TrimSuffix(s.Text(), ",")) + parts = append(parts, splitCSV(next)...) + } + + log.Name = trimQuotes(parts[0]) + log.State = strings.TrimPrefix(strings.TrimSpace(parts[1]), "CTLogState::") + log.Protocol = strings.TrimPrefix(strings.TrimSpace(parts[2]), "CTLogFormat::") + + // Next line: timestamp + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + tsLine := strings.TrimSpace(s.Text()) + tsValue := strings.Split(tsLine, ",")[0] + ts, err := strconv.ParseInt(strings.TrimSpace(tsValue), 10, 64) + if err != nil { + return log, err + } + log.Timestamp = time.Unix(0, ts*int64(time.Millisecond)) + + // Next line: operator index + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + opLine := strings.TrimSpace(s.Text()) + opValue := strings.Split(opLine, ",")[0] + opIndex, err := strconv.Atoi(strings.TrimSpace(opValue)) + if err != nil { + return log, err + } + log.OperatorIndex = opIndex + + // Key lines + var keyHex strings.Builder + for { + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + l := strings.TrimSpace(s.Text()) + if strings.HasPrefix(l, "\"") { + // remove trailing comma if any + trimmed := strings.TrimSuffix(l, ",") + keyHex.WriteString(trimQuotes(trimmed)) + if strings.HasSuffix(l, ",") { + // last key line + break + } + continue + } + return log, errors.New("unexpected line while reading key") + } + + key, err := decodeHexEscapes(keyHex.String()) + if err != nil { + return log, err + } + + // key length line + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + lenLine := strings.TrimSpace(s.Text()) + lenValue := strings.TrimSuffix(lenLine, "},") + keyLen, err := strconv.Atoi(strings.TrimSpace(lenValue)) + if err != nil { + return log, err + } + if len(key) != keyLen { + // ignore mismatch but continue + } + log.Key = key + return log, nil +} + +func readOperatorEntry(line string) (CTLogOperatorInfo, error) { + var op CTLogOperatorInfo + line = strings.TrimSuffix(strings.TrimSpace(line), ",") + if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") { + return op, errors.New("invalid operator entry") + } + line = strings.TrimPrefix(line, "{") + line = strings.TrimSuffix(line, "}") + parts := splitCSV(line) + if len(parts) != 2 { + return op, errors.New("invalid operator fields") + } + op.Name = trimQuotes(parts[0]) + id, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return op, err + } + op.ID = id + return op, nil +} + +func splitCSV(s string) []string { + var parts []string + var cur strings.Builder + inQuote := false + for i := 0; i < len(s); i++ { + c := s[i] + if c == '"' { + inQuote = !inQuote + cur.WriteByte(c) + continue + } + if c == ',' && !inQuote { + parts = append(parts, strings.TrimSpace(cur.String())) + cur.Reset() + continue + } + cur.WriteByte(c) + } + if cur.Len() > 0 { + parts = append(parts, strings.TrimSpace(cur.String())) + } + return parts +} + +func trimQuotes(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { + return s[1 : len(s)-1] + } + return s +} + +func decodeHexEscapes(s string) ([]byte, error) { + var out []byte + for i := 0; i < len(s); { + if i+3 >= len(s) || s[i] != '\\' || s[i+1] != 'x' { + return nil, errors.New("invalid escape") + } + b, err := strconv.ParseUint(s[i+2:i+4], 16, 8) + if err != nil { + return nil, err + } + out = append(out, byte(b)) + i += 4 + } + return out, nil +} diff --git a/third_party/certspotter/loglist/mozilla/ctlogs_test.go b/third_party/certspotter/loglist/mozilla/ctlogs_test.go new file mode 100644 index 0000000..3cc212e --- /dev/null +++ b/third_party/certspotter/loglist/mozilla/ctlogs_test.go @@ -0,0 +1,87 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package mozilla + +import ( + "crypto/sha256" + "encoding/base64" + "errors" + "net/http" + "testing" +) + +// parseFromURL downloads the CTKnownLogs.h file from the given URL and parses it. +func parseFromURL(url string) ([]CTLogInfo, []CTLogOperatorInfo, error) { + resp, err := http.Get(url) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, nil, errors.New(resp.Status) + } + return Parse(resp.Body) +} + +func TestParseFromURL(t *testing.T) { + logs, ops, err := parseFromURL("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h") + if err != nil { + t.Fatal(err) + } + if len(ops) == 0 { + t.Fatal("no operators parsed") + } + foundGoogle := false + foundSectigo := false + foundLets := false + for _, op := range ops { + if op.Name == "" { + t.Error("operator with empty name") + } + switch op.Name { + case "Google": + foundGoogle = true + case "Sectigo": + foundSectigo = true + case "Let's Encrypt": + foundLets = true + } + } + if !foundGoogle || !foundSectigo || !foundLets { + t.Errorf("missing expected operators: Google=%v Sectigo=%v Let's=%v", foundGoogle, foundSectigo, foundLets) + } + + if len(logs) == 0 { + t.Fatal("no logs parsed") + } + foundHash := false + targetHash := "1219ENGn9XfCx+lf1wC/+YLJM1pl4dCzAXMXwMjFaXc=" + for _, l := range logs { + if l.Name == "" { + t.Error("log with empty name") + } + if len(l.Key) == 0 { + t.Error("log with empty key") + } + if l.State != "Admissible" && l.State != "Retired" { + t.Errorf("unexpected state %q", l.State) + } + if l.Protocol != "RFC6962" && l.Protocol != "Tiled" { + t.Errorf("unexpected protocol %q", l.Protocol) + } + hash := sha256.Sum256(l.Key) + if base64.StdEncoding.EncodeToString(hash[:]) == targetHash { + foundHash = true + } + } + if !foundHash { + t.Errorf("log with key hash %s not found", targetHash) + } +} diff --git a/third_party/certspotter/loglist/mozilla/example_test.go b/third_party/certspotter/loglist/mozilla/example_test.go new file mode 100644 index 0000000..80888b5 --- /dev/null +++ b/third_party/certspotter/loglist/mozilla/example_test.go @@ -0,0 +1,55 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package mozilla_test + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + "log" + "net/http" + "os" + "text/tabwriter" + "time" + + "software.sslmate.com/src/certspotter/loglist/mozilla" +) + +func ExampleParse() { + resp, err := http.Get("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h") + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + log.Fatal(resp.Status) + } + logs, operators, err := mozilla.Parse(resp.Body) + if err != nil { + log.Fatal(err) + } + + tw := tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0) + fmt.Fprintln(tw, "Operator\tName") + for _, o := range operators { + fmt.Fprintf(tw, "%d\t%s\n", o.ID, o.Name) + } + tw.Flush() + + tw = tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0) + fmt.Fprintln(tw, "LogID\tProtocol\tState\tTimestamp\tOperator\tName") + for _, l := range logs { + hash := sha256.Sum256(l.Key) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%s\n", + base64.StdEncoding.EncodeToString(hash[:]), + l.Protocol, l.State, l.Timestamp.UTC().Format(time.RFC3339), l.OperatorIndex, l.Name) + } + tw.Flush() +} diff --git a/third_party/certspotter/loglist/schema.go b/third_party/certspotter/loglist/schema.go new file mode 100644 index 0000000..5dd2e04 --- /dev/null +++ b/third_party/certspotter/loglist/schema.go @@ -0,0 +1,158 @@ +// Copyright (C) 2020 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package loglist + +import ( + "time" + + "software.sslmate.com/src/certspotter/cttypes" +) + +type List struct { + Version string `json:"version,omitzero"` + LogListTimestamp time.Time `json:"log_list_timestamp,omitzero"` // Only present in v3 of schema + Operators []Operator `json:"operators,omitzero"` +} + +type Operator struct { + Name string `json:"name"` + Email []string `json:"email,omitzero"` + Logs []Log `json:"logs,omitzero"` + TiledLogs []Log `json:"tiled_logs,omitzero"` +} + +// Log represents the log object in the "v3" JSON log list schema, as published by Chrome, Apple, and SSLMate. +type Log struct { + Key []byte `json:"key"` + LogID cttypes.LogID `json:"log_id"` + MMD int `json:"mmd"` + URL string `json:"url,omitzero"` // only for rfc6962 logs + SubmissionURL string `json:"submission_url,omitzero"` // only for static-ct-api logs + MonitoringURL string `json:"monitoring_url,omitzero"` // only for static-ct-api logs + Description string `json:"description"` + State State `json:"state,omitzero"` + LogType LogType `json:"log_type,omitzero"` + TemporalInterval *struct { + StartInclusive time.Time `json:"start_inclusive"` + EndExclusive time.Time `json:"end_exclusive"` + } `json:"temporal_interval,omitzero"` + LogSoftware *struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"log_software,omitzero"` + + // certspotter-specific extensions + CertspotterDownloadSize int `json:"certspotter_download_size,omitzero"` + CertspotterDownloadJobs int `json:"certspotter_download_jobs,omitzero"` + CertspotterDownloadQPS float64 `json:"certspotter_download_qps,omitzero"` + + // TODO: add previous_operators +} + +func (log *Log) IsRFC6962() bool { return log.URL != "" } +func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" } + +// Return URL prefix for submission using the RFC6962 protocol +func (log *Log) GetSubmissionURL() string { + if log.SubmissionURL != "" { + return log.SubmissionURL + } else { + return log.URL + } +} + +// Return URL prefix for monitoring. +// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is +// only useful for informational purposes. +func (log *Log) GetMonitoringURL() string { + if log.MonitoringURL != "" { + return log.MonitoringURL + } else { + return log.URL + } +} + +type State struct { + Pending *struct { + Timestamp time.Time `json:"timestamp"` + } `json:"pending,omitzero"` + + Qualified *struct { + Timestamp time.Time `json:"timestamp"` + } `json:"qualified,omitzero"` + + Usable *struct { + Timestamp time.Time `json:"timestamp"` + } `json:"usable,omitzero"` + + Readonly *struct { + Timestamp time.Time `json:"timestamp"` + FinalTreeHead struct { + TreeSize int64 `json:"tree_size"` + SHA256RootHash []byte `json:"sha256_root_hash"` + } `json:"final_tree_head"` + } `json:"readonly,omitzero"` + + Retired *struct { + Timestamp time.Time `json:"timestamp"` + } `json:"retired,omitzero"` + + Rejected *struct { + Timestamp time.Time `json:"timestamp"` + } `json:"rejected,omitzero"` +} + +func (state *State) IsApproved() bool { + return state.Qualified != nil || state.Usable != nil || state.Readonly != nil +} + +func (state *State) WasApprovedAt(t time.Time) bool { + return state.Retired != nil && t.Before(state.Retired.Timestamp) +} + +type LogType string + +const ( + LogTypeProd = "prod" + LogTypeTest = "test" +) + +type LogSpec string + +const ( + StaticCTAPI = "static-ct-api" + RFC6962 = "rfc6962" +) + +// InclusionRequest represents the "v2" JSON log object, as published by log operators to request inclusion in browsers. +type InclusionRequest struct { + Key []byte `json:"key"` + LogID cttypes.LogID `json:"log_id"` + MMDSeconds int `json:"mmd_seconds"` + FriendlyName string `json:"friendly_name"` + LogSpec LogSpec `json:"log_spec"` + Endpoint struct { + URL string `json:"url"` + } `json:"endpoint,omitzero"` + SubmissionEndpoint struct { + URL string `json:"url"` + } `json:"submission_endpoint,omitzero"` + MonitoringEndpoint struct { + URL string `json:"url"` + } `json:"monitoring_endpoint,omitzero"` + TemporalInterval struct { + StartInclusive time.Time `json:"start_inclusive"` + EndExclusive time.Time `json:"end_exclusive"` + } `json:"temporal_interval"` + LogSoftware struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"log_software"` +} diff --git a/third_party/certspotter/loglist/validate.go b/third_party/certspotter/loglist/validate.go new file mode 100644 index 0000000..254846f --- /dev/null +++ b/third_party/certspotter/loglist/validate.go @@ -0,0 +1,57 @@ +// Copyright (C) 2020 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package loglist + +import ( + "crypto/sha256" + "fmt" +) + +func (list *List) Validate() error { + for i := range list.Operators { + if err := list.Operators[i].Validate(); err != nil { + return fmt.Errorf("problem with %dth operator (%s): %w", i, list.Operators[i].Name, err) + } + } + return nil +} + +func (operator *Operator) Validate() error { + for i := range operator.Logs { + if err := operator.Logs[i].Validate(); err != nil { + return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err) + } + } + for i := range operator.TiledLogs { + if err := operator.TiledLogs[i].Validate(); err != nil { + return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err) + } + } + return nil +} + +func (log *Log) Validate() error { + realLogID := sha256.Sum256(log.Key) + if log.LogID != realLogID { + return fmt.Errorf("log ID does not match log key") + } + + if !log.IsRFC6962() && !log.IsStaticCTAPI() { + return fmt.Errorf("URL(s) not provided") + } else if log.IsRFC6962() && log.IsStaticCTAPI() { + return fmt.Errorf("inconsistent URLs provided") + } + + if log.MMD < 0 { + return fmt.Errorf("log has a negative MMD") + } + + return nil +} diff --git a/third_party/certspotter/man/.gitignore b/third_party/certspotter/man/.gitignore new file mode 100644 index 0000000..8703c5c --- /dev/null +++ b/third_party/certspotter/man/.gitignore @@ -0,0 +1 @@ +*.8 diff --git a/third_party/certspotter/man/Makefile b/third_party/certspotter/man/Makefile new file mode 100644 index 0000000..8a7f575 --- /dev/null +++ b/third_party/certspotter/man/Makefile @@ -0,0 +1,9 @@ +all: certspotter-script.8 certspotter.8 certspotter-authorize.8 certspotter-metrics.8 + +%.8: %.md + lowdown -s -Tman \ + -M title:$(basename $(notdir $@)) \ + -M section:$(subst .,,$(suffix $@)) \ + -M date:$(if $(SOURCE_DATE_EPOCH),$(shell date -I -u -d "@$(SOURCE_DATE_EPOCH)"),$(shell date -I -u)) \ + -o $@ $< + diff --git a/third_party/certspotter/man/certspotter-authorize.md b/third_party/certspotter/man/certspotter-authorize.md new file mode 100644 index 0000000..ef320d9 --- /dev/null +++ b/third_party/certspotter/man/certspotter-authorize.md @@ -0,0 +1,105 @@ +# NAME + +**certspotter-authorize** - Authorize certificates to suppress certspotter notifications + +# SYNOPSIS + +**certspotter-authorize** [`-printhash`] *CERTFILE*... + +**certspotter-authorize** [`-printhash`] `-` + +# DESCRIPTION + +**certspotter-authorize** is a utility for preemptively authorizing +certificates so that **certspotter(8)** will not send notifications when +those certificates are discovered in Certificate Transparency logs. + +This is useful for preventing false alarms when you know in advance that a +certificate will be issued. For example, you might run **certspotter-authorize** +immediately after receiving a certificate from your certificate authority, as +part of your certificate issuance pipeline. + +**certspotter-authorize** uses the TBSCertificate hash as defined by RFC +6962 Section 3.2 to identify certificates. This hash is the same for a +certificate and its corresponding precertificate. This means authorizing +the certificate will suppress notifications for the precertificate +as well. Certificates with different serial numbers, validity periods, +or other changes to the TBSCertificate will not be covered by the authorization +and will trigger notifications. + +# ARGUMENTS + +**certspotter-authorize** takes paths to one or more PEM-encoded certificates +on the command line. If `-` is specified as the sole path, it reads the certificate +from stdin. + +# OPTIONS + +-state\_dir *PATH* + +: Directory where certspotter stores state. Defaults to + `$CERTSPOTTER_STATE_DIR` if set, or `~/.certspotter` otherwise. + This should be the same directory used by **certspotter(8)**. + +-printhash + +: Instead of authorizing the certificate, print its TBS hash (a hex-encoded SHA-256 digest) to stdout. + +-version + +: Print version information and exit. + +# EXAMPLES + +Authorize a certificate from a file: + + $ certspotter-authorize /path/to/cert.pem + +Authorize multiple certificates from files: + + $ certspotter-authorize cert-a.pem cert-b.pem + +Authorize a certificate from stdin: + + $ cat cert.pem | certspotter-authorize - + +Authorize a certificate in a custom state directory: + + $ certspotter-authorize -state_dir /var/lib/certspotter cert.pem + +Print the TBS hash without authorizing: + + $ certspotter-authorize -printhash /path/to/cert.pem + +# OPERATION + +**certspotter-authorize** computes the SHA-256 hash of each +certificate's TBSCertificate as defined by RFC 6962 Section 3.2 and +creates a `.notified` marker file in the certspotter +state directory. When certspotter later discovers a certificate with +the same TBSCertificate in a CT log, it will skip sending notifications +because the marker file is present. + +# ENVIRONMENT + +`CERTSPOTTER_STATE_DIR` + +: Directory for storing state. Overridden by `-state_dir`. Defaults to + `~/.certspotter`. This should be the same directory used by **certspotter(8)**. + +# FILES + +`$CERTSPOTTER_STATE_DIR/certs/XX/.HASH.notified` + +: Marker files indicating that a certificate with TBS hash `HASH` + has been authorized. `XX` is the first two hex digits of the hash. + The file is empty; only its existence is checked. + +# EXIT STATUS + +**certspotter-authorize** exits with status 0 on success, 1 on error, or 2 on +invalid usage. + +# SEE ALSO + +certspotter(8), certspotter-script(8) diff --git a/third_party/certspotter/man/certspotter-metrics.md b/third_party/certspotter/man/certspotter-metrics.md new file mode 100644 index 0000000..26aac49 --- /dev/null +++ b/third_party/certspotter/man/certspotter-metrics.md @@ -0,0 +1,128 @@ +# NAME + +**certspotter-metrics** - Output Prometheus metrics about certspotter's operation + +# SYNOPSIS + +**certspotter-metrics** [`-state_dir` *PATH*] + +# DESCRIPTION + +**certspotter-metrics** reads a certspotter state directory and writes +Prometheus metrics, in the Prometheus text exposition format, about +certspotter's progress monitoring each Certificate Transparency log. +Metrics are written to standard output. + +The intended use is to run **certspotter-metrics** periodically (e.g., from +cron) and expose its output to Prometheus, for example with the +prometheus-node-exporter(1) textfile collector. + +**certspotter-metrics** only reads the state directory; it never modifies it. +It is safe to run while certspotter is running, and it does not require +certspotter to be running. + +Metrics are output for every log for which certspotter has a state directory +(`$STATE_DIR/logs/LOG_ID`, where `LOG_ID` is the base64url-encoded log ID). +A log has a state directory once certspotter has started monitoring it. +The `log_id` label on each metric is the standard base64 encoding (with +padding) of the 32-byte log ID defined by RFC 6962. See +for a list of human-friendly names for each log ID. + +**certspotter-metrics** is experimental, and the metrics may change in a future release. + +# OPTIONS + +-state\_dir *PATH* + +: Directory where certspotter stores state. Defaults to + `$CERTSPOTTER_STATE_DIR` if set, or `~/.certspotter` otherwise. + This should be the same directory used by **certspotter(8)**. + +-version + +: Print version information and exit. + +# METRICS + +All metrics are gauges with a `log_id` label identifying the log the +metric pertains to. If a metric does not pertain to a log (because +certspotter has not stored the relevant state for the log yet), the log +does not have a sample for that metric. + +certspotter\_log\_max\_sth\_size + +: Tree size of the largest signed tree head (STH) certspotter has observed + from the log; in other words, the number of entries in the log. Absent + if certspotter has not yet downloaded an STH from the log. + +certspotter\_log\_max\_sth\_timestamp + +: Timestamp, in milliseconds since the Epoch, of the STH reported by + certspotter\_log\_max\_sth\_size. + +certspotter\_log\_download\_position + +: Number of entries certspotter has downloaded from the log, including + entries that have not yet been verified. + +certspotter\_log\_verified\_position + +: Number of entries certspotter has downloaded from the log and verified + against an STH. + +certspotter\_log\_verified\_sth\_timestamp + +: Timestamp, in milliseconds since the Epoch, of the most recent STH + certspotter has verified for the log. Absent if certspotter has not + verified an STH for the log. The size of this STH is equal to + certspotter\_log\_verified\_position. + +certspotter\_log\_unverified\_sths + +: Number of unverified STHs certspotter is currently storing for the log. + +certspotter\_log\_malformed\_entries + +: Number of malformed entries certspotter has recorded for the log. + certspotter records an entry as malformed when it cannot parse the + entry. See **certspotter(8)** for more information. + +certspotter\_log\_health\_check\_failures + +: Number of failed health checks certspotter has recorded for the log. + See **certspotter(8)** for information about health checks. + +# EXAMPLES + +crontab(5) entry that exposes metrics to the node\_exporter textfile collector +every five minutes (writing to a temporary file and renaming it +ensures that node\_exporter never reads a partially-written file): + + */5 * * * * root certspotter-metrics > /var/lib/node_exporter/certspotter.prom.tmp && mv /var/lib/node_exporter/certspotter.prom.tmp /var/lib/node_exporter/certspotter.prom + +Prometheus query to determine download backlog (number entries in the +log that haven't been downloaded yet): + + certspotter_log_max_sth_size-certspotter_log_download_position + +# ENVIRONMENT + +`CERTSPOTTER_STATE_DIR` + +: Directory for storing state. Overridden by `-state_dir`. Defaults to + `~/.certspotter`. This should be the same directory used by + **certspotter(8)**. + +`STATE_DIRECTORY` + +: Directory for storing state, if set and certspotter-metrics is running + under systemd. + +# EXIT STATUS + +**certspotter-metrics** exits with status 0 on success, 1 on error, or 2 on +invalid usage. + +# SEE ALSO + +certspotter(8) diff --git a/third_party/certspotter/man/certspotter-script.md b/third_party/certspotter/man/certspotter-script.md new file mode 100644 index 0000000..afe9fad --- /dev/null +++ b/third_party/certspotter/man/certspotter-script.md @@ -0,0 +1,296 @@ +# NAME + +**certspotter-script** - Certificate Transparency Log Monitor (hook script) + +# DESCRIPTION + +**certspotter-script** is *any* program that is executed by **certspotter(8)** +when it needs to notify you about an event, such as detecting a certificate for +a domain on your watch list. + +Scripts are placed in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory +(`~/.certspotter/hooks.d` by default), or specified on the command line +using the `-script` argument. + +# ENVIRONMENT + +## Event information + +The following environment variables are set for all types of events: + +`EVENT` + +: One of the following values, indicating the type of event: + + * `discovered_cert` - certspotter has discovered a certificate for a + domain on your watch list. + + * `malformed_cert` - certspotter can't determine if a certificate + matches your watch list because the certificate or the log entry + is malformed. + + * `error` - a problem is preventing certspotter from monitoring all + logs. + + Additional event types may be defined in the future, so your script should + be able to handle unknown values. + +`SUMMARY` + +: A short human-readable string describing the event. This is the same string + used in the subject line of emails sent by certspotter. + + +## Discovered certificate information + +The following environment variables are set for `discovered_cert` events: + +`WATCH_ITEM` + +: The item from your watch list which matches this certificate. + (If more than one item matches, the first one is used.) + +`LOG_URI` + +: The URI of the log containing the certificate. + +`ENTRY_INDEX` + +: The index of the log entry containing the certificate. + +`TBS_SHA256` + +: The hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2. + Certificates and their corresponding precertificates have the same `TBS_SHA256` value. + You can compute the TBS digest of a certificate using the `-printhash` option of the certspotter-authorize(8) command. + +`CERT_SHA256` + +: The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate or precertificate. + The digest is computed over the ASN.1 DER encoding. + +`PUBKEY_SHA256` + +: The hex-encoded SHA-256 digest of the certificate's Subject Public Key Info. + +`CERT_FILENAME` + +: Path to a file containing the PEM-encoded certificate chain. When `-no_save` is used, this points to a temporary file that is automatically deleted after your script completes. + +`JSON_FILENAME` + +: Path to a JSON file containing additional information about the certificate. See below for the format of the JSON file. + When `-no_save` is used, this points to a temporary file that is automatically deleted after your script completes. + +`TEXT_FILENAME` + +: Path to a text file containing information about the certificate. This file contains the same text that + certspotter uses in emails. You should not attempt to parse this file because its format may change in the future. + When `-no_save` is used, this points to a temporary file that is automatically deleted after your script completes. + +`NOT_BEFORE`, `NOT_BEFORE_UNIXTIME`, `NOT_BEFORE_RFC3339` + +: The not before time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively. These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set. + +`NOT_AFTER`, `NOT_AFTER_UNIXTIME`, `NOT_AFTER_RFC3339` + +: The not after (expiration) time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively. These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set. + +`VALIDITY_PARSE_ERROR` + +: Error parsing not before and not after, if any. If this variable is set, then the `NOT_BEFORE` and `NOT_AFTER` family of variables are unset. + +`SUBJECT_DN` + +: The distinguished name of the certificate's subject. This variable may be unset if there was a parse error, in which case `SUBJECT_PARSE_ERROR` is set. + +`SUBJECT_PARSE_ERROR` + +: Error parsing the subject, if any. If this variable is set, then `SUBJECT_DN` is unset. + +`ISSUER_DN` + +: The distinguished name of the certificate's issuer. This variable may be unset if there was a parse error, in which case `ISSUER_PARSE_ERROR` is set. Be aware that this value is controlled by the certificate authority and often contains [misleading, incorrect, or unhelpful information](https://www.agwa.name/blog/post/the_certificate_issuer_field_is_a_lie). + +`ISSUER_PARSE_ERROR` + +: Error parsing the issuer, if any. If this variable is set, then `ISSUER_DN` is unset. + +`SERIAL` + +: The hex-encoded serial number of the certificate. Prefixed with a minus (-) sign if negative. This variable may be unset if there was a parse error, in which case `SERIAL_PARSE_ERROR` is set. Be aware that malicious certificate authorities can issue certificates with duplicate serial numbers, so it is not safe to use the serial number to make security decisions (such as to determine if a certificate is legitimate or not). + +`SERIAL_PARSE_ERROR` + +: Error parsing the serial number, if any. If this variable is set, then `SERIAL` is unset. + +`CHAIN_ERROR` + +: Error building or verifying the certificate chain, if any. If this variable is set, then the certificate chain in `CERT_FILENAME` may be incomplete or invalid. + +## Malformed certificate information + +The following environment variables are set for `malformed_cert` events: + +`LOG_URI` + +: The URI of the log containing the malformed certificate. + +`ENTRY_INDEX` + +: The index of the log entry containing the malformed certificate. + +`LEAF_HASH` + +: The base64-encoded Merkle hash of the leaf containing the malformed certificate. + +`PARSE_ERROR` + +: A human-readable string describing why the certificate is malformed. + +`ENTRY_FILENAME` + +: Path to a file containing the JSON log entry. The file contains a JSON object with two fields, `leaf_input` and `extra_data`, as described in RFC 6962 Section 4.6. + +`TEXT_FILENAME` + +: Path to a text file containing a description of the malformed certificate. This file contains the same text that certspotter uses in emails. + +## Error information + +The following environment variables are set for `error` events: + +`TEXT_FILENAME` + +: Path to a text file containing a description of the error. This file contains the same text that certspotter uses in emails. + +# JSON FILE FORMAT + +certspotter saves a JSON file for every discovered certificate and puts the path to the file in `$JSON_FILENAME`. +When `-no_save` is used, the JSON file is created in a temporary location and automatically deleted after your script completes. +Otherwise, it is saved in the state directory for permanent storage. Your script can read the JSON file, such as with the jq(1) command, +to get additional information about the certificate which isn't appropriate for environment variables. + +The JSON file contains an object with the following fields: + +`tbs_sha256` + +: A string containing the hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2. + Certificates and their corresponding precertificates have the same `tbs_sha256` value. You can compute the + TBS digest of a certificate using the `-printhash` option of the certspotter-authorize(8) command. + +`pubkey_sha256` + +: A string containing the hex-encoded SHA-256 digest of the certificate's Subject Public Key Info. + +`dns_names` + +: An array of strings containing the DNS names for which the + certificate is valid, taken from both the DNS subject alternative names + (SANs) and the subject common name (CN). Internationalized domain names + are encoded in Punycode. + +`ip_addresses` + +: An array of strings containing the IP addresses for which the certificate is valid, + taken from both the IP subject alternative names (SANs) and the subject common name (CN). + +`not_before` + +: A string containing the not before time of the certificate in RFC3339 format. + Null if there was an error parsing the certificate's validity. + +`not_after` + +: A string containing the not after (expiration) time of the certificate in RFC3339 format. + Null if there was an error parsing the certificate's validity. + +Additional fields will be added in the future based on user feedback. Please open +an issue at if you have a use case for another field. + +# DETERMINING IF A CERTIFICATE IS AUTHORIZED + +You may want to compare a discovered certificate against a database of legitimate certificates +to decide whether to raise an alarm about it. Beware the following caveats: + + * Since certificates may be logged to CT in the form of a precertificate, it doesn't work + to compare the discovered certificate's fingerprint against a list of legitimate fingerprints. + + * Since malicious certificate authorities can issue certificates with duplicate serial numbers, + it is not secure to compare the the discovered certificate's serial number against a list of + legitimate serial numbers. + + * Since malicious certificate authorities can put any value in a certificate's Subject Key Identifier (SKI) + extension, it is not secure to compare a discovered certificate's Subject Key Identifier against + a list of legitimate Subject Key Identifiers. + + * Since malicious certificate authorities can put any value in the certificate's Issuer field, + it is not secure to compare a discovered certificate's Issuer against a list of authorized Issuers. + +To determine if a certificate is legitimate, you should do one of the following: + + * Compare the discovered certificate's TBS hash (as provided in the `$TBS_SHA256` variable) against + a list of authorized TBS hashes. The TBS hash is computed over the DER encoding of the certificate's + TBSCertificate, *with any SCT extension removed*. You can use the `-printhash` option + of the certspotter-authorize(8) command to compute a certificate's TBS hash. + + * Compare the discovered certificate's public key hash (as provided in the `$PUBKEY_SHA256` variable) against + a list of authorized public key hashes. The public key hash is computed over the DER encoding of the + certificate's Subject Public Key Info. You can use the following openssl command to compute a certificate's + public key hash: `openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | openssl sha256` + + +# EXAMPLES + +Example environment variables for a `discovered_cert` event: + +``` +CERT_FILENAME=/home/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.pem +CERT_SHA256=3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a +ENTRY_INDEX=6464843 +EVENT=discovered_cert +ISSUER_DN=C=GB, ST=Greater Manchester, L=Salford, O=Sectigo Limited, CN=Sectigo RSA Domain Validation Secure Server CA +JSON_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.v1.json +LOG_URI=https://ct.cloudflare.com/logs/nimbus2024/ +NOT_AFTER='2024-01-26 03:47:26 +0000 UTC' +NOT_AFTER_RFC3339=2024-01-26T03:47:26Z +NOT_AFTER_UNIXTIME=1706240846 +NOT_BEFORE='2023-01-31 03:47:26 +0000 UTC' +NOT_BEFORE_RFC3339=2023-01-31T03:47:26Z +NOT_BEFORE_UNIXTIME=1675136846 +PUBKEY_SHA256=33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8 +SERIAL=c170fbf3bf27481e5c351a4db6f2dc5f +SUBJECT_DN=CN=sslmate.com +SUMMARY='certificate discovered for .sslmate.com' +TBS_SHA256=2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8 +TEXT_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.txt +WATCH_ITEM=.sslmate.com +``` + +Example JSON file for a discovered certificate: + +``` +{ + "dns_names": [ + "sslmate.com", + "www.sslmate.com" + ], + "ip_addresses": [], + "not_after": "2024-01-26T03:47:26Z", + "not_before": "2023-01-31T03:47:26Z", + "pubkey_sha256": "33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8", + "tbs_sha256": "2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8" +} +``` + +# SEE ALSO + +certspotter(8) + +# COPYRIGHT + +Copyright (c) 2016-2025 Opsmate, Inc. + +# BUGS + +Report bugs to . diff --git a/third_party/certspotter/man/certspotter.md b/third_party/certspotter/man/certspotter.md new file mode 100644 index 0000000..ecbe9c0 --- /dev/null +++ b/third_party/certspotter/man/certspotter.md @@ -0,0 +1,261 @@ +# NAME + +**certspotter** - Certificate Transparency Log Monitor + +# SYNOPSIS + +**certspotter** [`-start_at_end`] [`-watchlist` *FILENAME*] [`-email` *ADDRESS*] `...` + +# DESCRIPTION + +**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that +alerts you when a SSL/TLS certificate is issued for one of your domains. +Cert Spotter is easier to use than other open source CT monitors, since it does not require +a database. It's also more robust, since it uses a special certificate parser +that ensures it won't miss certificates. + +Cert Spotter is also available as a hosted service by SSLMate, +. + +You can use Cert Spotter to detect: + + * Certificates issued to attackers who have compromised your DNS and + are redirecting your visitors to their malicious site. + * Certificates issued to attackers who have taken over an abandoned + sub-domain in order to serve malware under your name. + * Certificates issued to attackers who have compromised a certificate + authority and want to impersonate your site. + * Certificates issued in violation of your corporate policy + or outside of your centralized certificate procurement process. + +# OPTIONS + +-email *ADDRESS* + +: Email address to contact when a matching certificate is discovered, or + an error occurs. You can specify this option more than once to email + multiple addresses. Your system must have a working sendmail(1) command. + + Regardless of the `-email` option, certspotter also emails any address listed + in `$CERTSPOTTER_CONFIG_DIR/email_recipients` file + (`~/.certspotter/email_recipients` by default). (One address per line, + blank lines are ignored.) This file is read only at startup, so you + must restart certspotter if you change it. + +-healthcheck *INTERVAL* + +: Perform a health check at the given interval (default: "24h") as described + below. *INTERVAL* must be a decimal number followed by "h" for hours or + "m" for minutes. + +-logs *ADDRESS* + +: Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor. + The schema for this file can be found at . + Defaults to , which includes + the union of active logs recognized by Chrome and Apple. certspotter loads the + log list when starting up, and periodically reloads it in case it has changed. + +-no\_save + +: Do not save a copy of matching certificates. Note that enabling this option + will cause you to receive duplicate notifications, since certspotter will + have no way of knowing if you've been previously notified about a certificate. + +-script *COMMAND* + +: Command to execute when a matching certificate is found or an error occurs. See + certspotter-script(8) for information about the interface to scripts. + + Regardless of the `-script` option, certspotter also executes any executable + file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory + (`~/.certspotter/hooks.d` by default). + +-start\_at\_end + +: Start monitoring logs from the end rather than the beginning. + + **WARNING**: monitoring from the beginning guarantees detection of all + certificates, but requires downloading hundreds of millions of + certificates, which takes days. + +-state\_dir *PATH* + +: Directory for storing state. Defaults to `$CERTSPOTTER_STATE_DIR`, which is + "~/.certspotter" by default. + +-stdout + +: Write matching certificates and errors to stdout. + +-verbose + +: Print detailed information about certspotter's operation to stderr. + +-version + +: Print version and exit. + +-watchlist *PATH* + +: File containing DNS names to monitor, one per line. To monitor an entire + domain namespace (including the domain itself and all sub-domains) prefix + the domain name with a dot (e.g. ".example.com"). To monitor a single DNS + name only, do not prefix the name with a dot. + + Defaults to `$CERTSPOTTER_CONFIG_DIR/watchlist`, which is + "~/.certspotter/watchlist" by default. + Specify `-` to read the watch list from stdin. + + certspotter reads the watch list only when starting up, so you must restart + certspotter if you change it. + +# NOTIFICATIONS + +When certspotter detects a certificate matching your watchlist, or encounters +an error that is preventing it from discovering certificates, it notifies you +as follows: + +* Emails any address specified by the `-email` command line flag. + +* Emails any address listed in the `$CERTSPOTTER_CONFIG_DIR/email_recipients` + file (`~/.certspotter/email_recipients` by default). (One address per line, + blank lines are ignored.) This file is read only at startup, so you + must restart certspotter if you change it. + +* Executes the script specified by the `-script` command line flag. + +* Executes every executable file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` + directory (`~/.certspotter/hooks.d` by default). + +* Writes the notification to standard out if the `-stdout` flag was specified. + +Sending email requires a working sendmail(1) command. For details about +the script interface, see certspotter-script(8). + +# OPERATION + +certspotter continuously monitors all browser-recognized Certificate +Transparency logs (both RFC6962 and static-ct-api) looking for certificates (including precertificates) +which are valid for any domain on your watch list. When certspotter +detects a matching certificate, it emails you, executes a script, and/or +writes a report to standard out, as described above. + +certspotter also saves a copy of matching certificates in +`$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default) +unless you specify the `-no_save` option. + +When certspotter has not previously monitored a log, it can either start +monitoring the log from the beginning, or seek to the end of the log and +start monitoring from there. Monitoring from the beginning guarantees +detection of all certificates, but requires downloading hundreds of +millions of certificates, which takes days. The default behavior is to +monitor from the beginning. To start monitoring new logs from the end, +specify the `-start_at_end` option. + +If certspotter has previously monitored a log, it resumes monitoring +the log from the previous position. This means that if you add +a domain to your watch list, certspotter will not detect any certificates +that were logged prior to the addition. To detect such certificates, +you must delete `$CERTSPOTTER_STATE_DIR/logs`, which will cause certspotter +to restart monitoring from the very beginning of each log (provided +`-start_at_end` is not specified). This will cause certspotter to download +hundreds of millions of certificates, which takes days. To find preexisting +certificates, it's faster to use the Cert Spotter service +, SSLMate's Certificate Transparency Search +API , or a CT search engine such as +. + +# ERROR HANDLING + +When certspotter encounters a problem with the local system (e.g. failure +to write a file, send an email, or execute a script), it prints a message to stderr and +exits with a non-zero status. + +When certspotter encounters a problem loading the log list during startup, it +prints a message to stderr and exits with a non-zero status. When certspotter encounters a problem +reloading the log list, it prints a message to stderr and continues running with the previously-loaded +log list. It will try reloading the log list again later. + +When certspotter encounters a problem contacting a log, it writes the error to a file in +the state directory and continues running. It will try contacting the log again later; +most log errors are transient. The last 7 days of errors are kept. + +Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the +following health checks: + + * Ensure that the log list has been successfully retrieved at least once + since the previous health check. + * Ensure that every log has been successfully contacted at least once + since the previous health check. + * Ensure that certspotter is not falling behind monitoring any logs. + +If any health check fails, certspotter notifies you by email, script, and/or +standard out, as described above. The notification includes the last several errors +encountered when contacting the log. + +Health check failures should be rare, and you should take them seriously because it means +certspotter might not detect all certificates. It might also be an indication +of CT log misbehavior. Check the error files for details, and if +you need help, file an issue at . + +# EXIT STATUS + +certspotter exits 0 when it receives `SIGTERM` or `SIGINT`, +and non-zero when a serious error occurs. + +# ENVIRONMENT + +`CERTSPOTTER_STATE_DIR` + +: Directory for storing state. Overridden by `-state_dir`. Defaults to + `~/.certspotter`. + +`CERTSPOTTER_CONFIG_DIR` + +: Directory from which any configuration, such as the watch list, is read. + Defaults to `~/.certspotter`. + +`EMAIL` + +: Email address from which to send emails. If not set, certspotter lets sendmail pick + the address. + +`HTTPS_PROXY` + +: URL of proxy server for making HTTPS requests. `http://`, `https://`, and + `socks5://` URLs are supported. By default, no proxy server is used. + +`SENDMAIL_PATH` + +: Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`. + +If the `SYSTEMD_EXEC_PID` environment variable equals the PID of certspotter, then certspotter assumes it is running under systemd and respects the `CONFIGURATION_DIRECTORY`, `STATE_DIRECTORY`, and `CACHE_DIRECTORY` environment variables, unless overridden by a certspotter-specific environment variable or flag. + +certspotter respects the `XDG_CACHE_HOME` environment variable for locating its cache directory. For historical reasons, certspotter does not respect `XDG_CONFIG_HOME` or `XDG_STATE_HOME`. + +# DIRECTORIES + +Config directory + +: Stores configuration, such as the watch list. The location is: (1) the `CERTSPOTTER_CONFIG_DIR` environment variable, if set, (2) the `CONFIGURATION_DIRECTORY` environment variable, if set and certspotter is running under systemd, or (3) the default location `~/.certspotter`. certspotter does not write to this directory. + +State directory + +: Stores state, such as the position of each log and a store of discovered certificates. The location is: (1) the `-state_dir` command line flag, if provided, (2) the `CERTSPOTTER_STATE_DIR` environment variable, if set, (3) the `STATE_DIRECTORY` environment variable, if set and certspotter is running under systemd, or (4) the default location `~/.certspotter`. certspotter creates this directory if necessary. + +Cache directory + +: Stores cached data. The location is (1) the `CERTSPOTTER_CACHE_DIR` environment variable if set, (2) the `CACHE_DIRECTORY` environment, if set and certspotter is running under systemd, or (3) `$XDG_CACHE_HOME/certspotter` (which on Linux is `~/.cache/certspotter` by default). You can delete this directory without impacting functionality, but certspotter may need to perform additional computation or network requests. + +# SEE ALSO + +certspotter-script(8) + +# COPYRIGHT + +Copyright (c) 2016-2025 Opsmate, Inc. + +# BUGS + +Report bugs to . diff --git a/third_party/certspotter/merkletree/collapsed_tree.go b/third_party/certspotter/merkletree/collapsed_tree.go new file mode 100644 index 0000000..dd3e586 --- /dev/null +++ b/third_party/certspotter/merkletree/collapsed_tree.go @@ -0,0 +1,182 @@ +// Copyright (C) 2022 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "encoding/json" + "fmt" + "math/bits" + "slices" +) + +// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending +// nodes and calculating the root hash. +type CollapsedTree struct { + offset uint64 + nodes []Hash + size uint64 +} + +func calculateNumNodes(size uint64) int { + return bits.OnesCount64(size) +} + +// TODO: phase out this function +func EmptyCollapsedTree() *CollapsedTree { + return &CollapsedTree{nodes: []Hash{}, size: 0} +} + +// TODO: phase out this function +func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { + tree := new(CollapsedTree) + if err := tree.Init(nodes, size); err != nil { + return nil, err + } + return tree, nil +} + +func (tree CollapsedTree) Equal(other CollapsedTree) bool { + return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes) +} + +func (tree CollapsedTree) Clone() CollapsedTree { + return CollapsedTree{ + offset: tree.offset, + nodes: slices.Clone(tree.nodes), + size: tree.size, + } +} + +// Add a new leaf hash to the end of the tree. +// Returns an error if and only if the new tree would be too large for the subtree offset. +// Always returns a nil error if tree.Offset() == 0. +func (tree *CollapsedTree) Add(hash Hash) error { + if tree.offset > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(tree.offset) + if tree.size+1 > maxSize { + return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize) + } + } + tree.nodes = append(tree.nodes, hash) + tree.size++ + tree.collapse() + return nil +} + +func (tree *CollapsedTree) Append(other CollapsedTree) error { + if tree.offset+tree.size != other.offset { + return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size) + } + if tree.offset > 0 { + newSize := tree.size + other.size + maxSize := uint64(1) << bits.TrailingZeros64(tree.offset) + if newSize > maxSize { + return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset) + } + } + if tree.size > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(tree.size) + if other.size > maxSize { + return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize) + } + } + + tree.nodes = append(tree.nodes, other.nodes...) + tree.size += other.size + tree.collapse() + return nil +} + +func (tree *CollapsedTree) collapse() { + numNodes := calculateNumNodes(tree.size) + for len(tree.nodes) > numNodes { + left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1] + tree.nodes = tree.nodes[:len(tree.nodes)-2] + tree.nodes = append(tree.nodes, HashChildren(left, right)) + } +} + +func (tree CollapsedTree) CalculateRoot() Hash { + if len(tree.nodes) == 0 { + return HashNothing() + } + i := len(tree.nodes) - 1 + hash := tree.nodes[i] + for i > 0 { + i -= 1 + hash = HashChildren(tree.nodes[i], hash) + } + return hash +} + +// Return the subtree offset (0 if this represents an entire tree) +func (tree CollapsedTree) Offset() uint64 { + return tree.offset +} + +// Return a non-nil slice containing the nodes. The slice +// must not be modified. +func (tree CollapsedTree) Nodes() []Hash { + if tree.nodes == nil { + return []Hash{} + } else { + return tree.nodes + } +} + +// Return the number of leaf nodes in the tree. +func (tree CollapsedTree) Size() uint64 { + return tree.size +} + +type collapsedTreeMessage struct { + Offset uint64 `json:"offset,omitzero"` + Nodes []Hash `json:"nodes"` // never nil + Size uint64 `json:"size"` +} + +func (tree CollapsedTree) MarshalJSON() ([]byte, error) { + return json.Marshal(collapsedTreeMessage{ + Offset: tree.offset, + Nodes: tree.Nodes(), + Size: tree.size, + }) +} + +func (tree *CollapsedTree) UnmarshalJSON(b []byte) error { + var rawTree collapsedTreeMessage + if err := json.Unmarshal(b, &rawTree); err != nil { + return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) + } + if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil { + return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) + } + return nil +} + +func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error { + if len(nodes) != calculateNumNodes(size) { + return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes)) + } + tree.size = size + tree.nodes = nodes + return nil +} + +func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error { + if offset > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(offset) + if size > maxSize { + return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize) + } + } + tree.offset = offset + return tree.Init(nodes, size) +} diff --git a/third_party/certspotter/merkletree/fragment.go b/third_party/certspotter/merkletree/fragment.go new file mode 100644 index 0000000..1b0f0a4 --- /dev/null +++ b/third_party/certspotter/merkletree/fragment.go @@ -0,0 +1,122 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "encoding/json" + "fmt" + "slices" +) + +// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees +type FragmentedCollapsedTree struct { + subtrees []CollapsedTree // sorted by offset +} + +func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error { + return tree.Add(CollapsedTree{ + offset: position, + nodes: []Hash{hash}, + size: 1, + }) +} + +func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error { + if subtree.size == 0 { + return nil + } + i := len(tree.subtrees) + for i > 0 && tree.subtrees[i-1].offset > subtree.offset { + i-- + } + if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset { + return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size) + } + if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset { + return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset) + } + if i == 0 || tree.subtrees[i-1].Append(subtree) != nil { + tree.subtrees = slices.Insert(tree.subtrees, i, subtree) + i++ + } + for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil { + tree.subtrees = slices.Delete(tree.subtrees, i, i+1) + } + return nil +} + +func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error { + // TODO: try to make this linear time instead of quadratic; it should be possible since the subtrees are sorted by offset + for _, subtree := range other.subtrees { + if err := tree.Add(subtree); err != nil { + return err + } + } + return nil +} + +func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) { + var prevEnd uint64 + for i := range tree.subtrees { + if prevEnd != tree.subtrees[i].offset { + if !yield(prevEnd, tree.subtrees[i].offset) { + return + } + } + prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size + } + yield(prevEnd, 0) +} + +func (tree FragmentedCollapsedTree) NumSubtrees() int { + return len(tree.subtrees) +} + +func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree { + return tree.subtrees[i] +} + +func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree { + if tree.subtrees == nil { + return []CollapsedTree{} + } else { + return tree.subtrees + } +} + +// Return true iff the tree contains at least the first n nodes (without any gaps) +func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool { + return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n +} + +func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error { + for i := 1; i < len(subtrees); i++ { + if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset { + return fmt.Errorf("subtrees %d and %d overlap", i-1, i) + } + } + tree.subtrees = subtrees + return nil +} + +func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) { + return json.Marshal(tree.Subtrees()) +} + +func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error { + var subtrees []CollapsedTree + if err := json.Unmarshal(b, &subtrees); err != nil { + return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err) + } + if err := tree.Init(subtrees); err != nil { + return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err) + } + return nil +} diff --git a/third_party/certspotter/merkletree/hash.go b/third_party/certspotter/merkletree/hash.go new file mode 100644 index 0000000..2666ba0 --- /dev/null +++ b/third_party/certspotter/merkletree/hash.go @@ -0,0 +1,77 @@ +// Copyright (C) 2022 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" +) + +const HashLen = 32 + +type Hash [HashLen]byte + +func (h Hash) Compare(other Hash) int { + return bytes.Compare(h[:], other[:]) +} + +func (h Hash) Base64String() string { + return base64.StdEncoding.EncodeToString(h[:]) +} + +func (h Hash) MarshalJSON() ([]byte, error) { + return json.Marshal(h[:]) +} + +func (h Hash) MarshalBinary() ([]byte, error) { + return h[:], nil +} + +func (h *Hash) UnmarshalJSON(b []byte) error { + var hashBytes []byte + if err := json.Unmarshal(b, &hashBytes); err != nil { + return err + } + return h.UnmarshalBinary(hashBytes) +} + +func (h *Hash) UnmarshalBinary(hashBytes []byte) error { + if len(hashBytes) != HashLen { + return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes)) + } + copy(h[:], hashBytes) + return nil +} + +func HashNothing() Hash { + return sha256.Sum256(nil) +} + +func HashLeaf(leafBytes []byte) Hash { + var hash Hash + hasher := sha256.New() + hasher.Write([]byte{0x00}) + hasher.Write(leafBytes) + hasher.Sum(hash[:0]) + return hash +} + +func HashChildren(left Hash, right Hash) Hash { + var hash Hash + hasher := sha256.New() + hasher.Write([]byte{0x01}) + hasher.Write(left[:]) + hasher.Write(right[:]) + hasher.Sum(hash[:0]) + return hash +} diff --git a/third_party/certspotter/monitor/config.go b/third_party/certspotter/monitor/config.go new file mode 100644 index 0000000..9d2fb31 --- /dev/null +++ b/third_party/certspotter/monitor/config.go @@ -0,0 +1,39 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "time" + + "software.sslmate.com/src/certspotter" +) + +// IdentifierSink receives the DNS identifiers parsed from every monitored +// certificate. This is a CTCLOUD PATCH to upstream certspotter; see +// CTCLOUD_PATCH.md. When non-nil it is invoked on the monitoring hot path +// instead of the watchlist/notification machinery. +type IdentifierSink interface { + HandleIdentifiers(ctx context.Context, identifiers *certspotter.Identifiers) +} + +type Config struct { + LogListSource string + State StateProvider + StartAtEnd bool + WatchList WatchList + Verbose bool + HealthCheckInterval time.Duration + + // IdentifierSink is a CTCLOUD PATCH (see CTCLOUD_PATCH.md). When set, + // parsed identifiers are handed directly to it in-process and the normal + // watchlist/chain/notification path is skipped. + IdentifierSink IdentifierSink +} diff --git a/third_party/certspotter/monitor/daemon.go b/third_party/certspotter/monitor/daemon.go new file mode 100644 index 0000000..d1b8456 --- /dev/null +++ b/third_party/certspotter/monitor/daemon.go @@ -0,0 +1,173 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "errors" + "fmt" + "golang.org/x/sync/errgroup" + "log" + insecurerand "math/rand" + "software.sslmate.com/src/certspotter/loglist" + "time" +) + +const ( + reloadLogListIntervalMin = 30 * time.Minute + reloadLogListIntervalMax = 90 * time.Minute +) + +func randomDuration(min, max time.Duration) time.Duration { + return min + time.Duration(insecurerand.Int63n(int64(max-min+1))) +} + +func reloadLogListInterval() time.Duration { + return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax) +} + +type task struct { + log *loglist.Log + stop context.CancelFunc +} + +type daemon struct { + config *Config + taskgroup *errgroup.Group + tasks map[LogID]task + logsLoadedAt time.Time + logListToken *loglist.ModificationToken +} + +func (daemon *daemon) healthCheck(ctx context.Context) error { + if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval { + errors, err := daemon.config.State.GetErrors(ctx, nil, recentErrorCount) + if err != nil { + return fmt.Errorf("error getting recent errors: %w", err) + } + var errorsDir string + if fsstate, ok := daemon.config.State.(*FilesystemState); ok { + errorsDir = fsstate.errorDir(nil) + } + info := &StaleLogListInfo{ + Source: daemon.config.LogListSource, + LastSuccess: daemon.logsLoadedAt, + RecentErrors: errors, + ErrorsDir: errorsDir, + } + if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil { + return fmt.Errorf("error notifying about stale log list: %w", err) + } + } + + for _, task := range daemon.tasks { + if err := healthCheckLog(ctx, daemon.config, task.log); err != nil { + return fmt.Errorf("error checking health of log %q: %w", task.log.GetMonitoringURL(), err) + } + } + return nil +} + +func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task { + ctx, cancel := context.WithCancel(ctx) + daemon.taskgroup.Go(func() error { + defer cancel() + err := monitorLogContinously(ctx, daemon.config, ctlog) + if daemon.config.Verbose { + log.Printf("%s: task stopped with error: %s", ctlog.GetMonitoringURL(), err) + } + if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) { + return nil + } else { + return fmt.Errorf("error while monitoring %s: %w", ctlog.GetMonitoringURL(), err) + } + }) + return task{log: ctlog, stop: cancel} +} + +func (daemon *daemon) loadLogList(ctx context.Context) error { + newLogList, newToken, err := getLogList(ctx, daemon.config.LogListSource, daemon.logListToken) + if errors.Is(err, loglist.ErrNotModified) { + daemon.logsLoadedAt = time.Now() // so the log list is not reported as stale + return nil + } else if err != nil { + return err + } + + if daemon.config.Verbose { + log.Printf("fetched %d logs from %q", len(newLogList), daemon.config.LogListSource) + } + + for logID, task := range daemon.tasks { + if _, exists := newLogList[logID]; exists { + continue + } + if daemon.config.Verbose { + log.Printf("stopping task for log %s", logID.Base64String()) + } + task.stop() + delete(daemon.tasks, logID) + } + for logID, ctlog := range newLogList { + if _, isRunning := daemon.tasks[logID]; isRunning { + continue + } + if daemon.config.Verbose { + log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.GetMonitoringURL()) + } + daemon.tasks[logID] = daemon.startTask(ctx, ctlog) + } + daemon.logsLoadedAt = time.Now() + daemon.logListToken = newToken + return nil +} + +func (daemon *daemon) run(ctx context.Context) error { + if err := daemon.config.State.Prepare(ctx); err != nil { + return fmt.Errorf("error preparing state: %w", err) + } + + if err := daemon.loadLogList(ctx); err != nil { + return fmt.Errorf("error loading log list: %w", err) + } + + reloadLogListTicker := time.NewTicker(reloadLogListInterval()) + defer reloadLogListTicker.Stop() + + healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval) + defer healthCheckTicker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-reloadLogListTicker.C: + if err := daemon.loadLogList(ctx); err != nil { + recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err)) + } + reloadLogListTicker.Reset(reloadLogListInterval()) + case <-healthCheckTicker.C: + if err := daemon.healthCheck(ctx); err != nil { + return err + } + } + } +} + +func Run(ctx context.Context, config *Config) error { + group, ctx := errgroup.WithContext(ctx) + daemon := &daemon{ + config: config, + taskgroup: group, + tasks: make(map[LogID]task), + } + group.Go(func() error { return daemon.run(ctx) }) + return group.Wait() +} diff --git a/third_party/certspotter/monitor/discoveredcert.go b/third_party/certspotter/monitor/discoveredcert.go new file mode 100644 index 0000000..a2e99e9 --- /dev/null +++ b/third_party/certspotter/monitor/discoveredcert.go @@ -0,0 +1,189 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "encoding/hex" + "encoding/pem" + "fmt" + "strings" + "time" + + "software.sslmate.com/src/certspotter" + "software.sslmate.com/src/certspotter/cttypes" +) + +type DiscoveredCert struct { + WatchItem WatchItem + LogEntry *LogEntry + Info *certspotter.CertInfo + Chain []cttypes.ASN1Cert // first entry is the leaf certificate or precertificate + ChainError error // any error generating or validating Chain; if non-nil, Chain may be partial or incorrect + TBSSHA256 [32]byte // computed over Info.TBS.Raw + SHA256 [32]byte // computed over Chain[0] + PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes + Identifiers *certspotter.Identifiers +} + +type certPaths struct { + temporary bool + + certPath string + jsonPath string + textPath string +} + +func (cert *DiscoveredCert) pemChain() []byte { + var buffer bytes.Buffer + if cert.ChainError != nil { + fmt.Fprintln(&buffer, "Warning: this chain may be incomplete or invalid: ", cert.ChainError) + } + for _, certBytes := range cert.Chain { + if err := pem.Encode(&buffer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }); err != nil { + panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err)) + } + } + return buffer.Bytes() +} + +func (cert *DiscoveredCert) json() any { + object := map[string]any{ + "tbs_sha256": hex.EncodeToString(cert.TBSSHA256[:]), + "pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]), + "dns_names": cert.Identifiers.DNSNames, + "ip_addresses": cert.Identifiers.IPAddrs, + } + + if cert.Info.ValidityParseError == nil { + object["not_before"] = cert.Info.Validity.NotBefore + object["not_after"] = cert.Info.Validity.NotAfter + } else { + object["not_before"] = nil + object["not_after"] = nil + } + + return object +} + +func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error { + if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil { + return err + } + if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil { + return err + } + if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil { + return err + } + return nil +} + +func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string { + env := []string{ + "EVENT=discovered_cert", + "SUMMARY=" + certNotificationSummary(cert), + "CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented + "LOG_URI=" + cert.LogEntry.Log.GetMonitoringURL(), + "ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index), + "WATCH_ITEM=" + cert.WatchItem.String(), + "TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]), + "CERT_SHA256=" + hex.EncodeToString(cert.SHA256[:]), + "FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented + "PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]), + "PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented + } + + if paths != nil { + env = append(env, "CERT_FILENAME="+paths.certPath) + env = append(env, "JSON_FILENAME="+paths.jsonPath) + env = append(env, "TEXT_FILENAME="+paths.textPath) + } + + if cert.Info.ValidityParseError == nil { + env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String()) + env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix())) + env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339)) + env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String()) + env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix())) + env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339)) + } else { + env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error()) + } + + if cert.Info.SubjectParseError == nil { + env = append(env, "SUBJECT_DN="+cert.Info.Subject.String()) + } else { + env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error()) + } + + if cert.Info.IssuerParseError == nil { + env = append(env, "ISSUER_DN="+cert.Info.Issuer.String()) + } else { + env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error()) + } + + if cert.Info.SerialNumberParseError == nil { + env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber)) + } else { + env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error()) + } + + if cert.ChainError != nil { + env = append(env, "CHAIN_ERROR="+cert.ChainError.Error()) + } + + return env +} + +func certNotificationText(cert *DiscoveredCert, paths *certPaths) string { + // TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration) + + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + + fmt.Fprintf(text, "%x:\n", cert.SHA256) + for _, dnsName := range cert.Identifiers.DNSNames { + writeField("DNS Name", dnsName) + } + for _, ipaddr := range cert.Identifiers.IPAddrs { + writeField("IP Address", ipaddr) + } + writeField("Pubkey", hex.EncodeToString(cert.PubkeySHA256[:])) + if cert.Info.IssuerParseError == nil { + writeField("Issuer", cert.Info.Issuer) + } else { + writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError)) + } + if cert.Info.ValidityParseError == nil { + writeField("Not Before", cert.Info.Validity.NotBefore) + writeField("Not After", cert.Info.Validity.NotAfter) + } else { + writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) + writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) + } + writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.GetMonitoringURL())) + writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:])) + if cert.ChainError != nil { + writeField("Error Building Chain", cert.ChainError.Error()) + } + if paths != nil && !paths.temporary { + writeField("Filename", paths.certPath) + } + + return text.String() +} + +func certNotificationSummary(cert *DiscoveredCert) string { + return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem) +} diff --git a/third_party/certspotter/monitor/errors.go b/third_party/certspotter/monitor/errors.go new file mode 100644 index 0000000..ca12889 --- /dev/null +++ b/third_party/certspotter/monitor/errors.go @@ -0,0 +1,28 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "log" + + "software.sslmate.com/src/certspotter/loglist" +) + +func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) { + if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil { + log.Print("unable to notify about error: ", err) + if ctlog == nil { + log.Print(errToRecord) + } else { + log.Print(ctlog.GetMonitoringURL(), ": ", errToRecord) + } + } +} diff --git a/third_party/certspotter/monitor/fileutils.go b/third_party/certspotter/monitor/fileutils.go new file mode 100644 index 0000000..96e5b90 --- /dev/null +++ b/third_party/certspotter/monitor/fileutils.go @@ -0,0 +1,138 @@ +// Copyright (C) 2017, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "slices" + "strings" +) + +func randomFileSuffix() string { + var randomBytes [12]byte + if _, err := rand.Read(randomBytes[:]); err != nil { + panic(err) + } + return hex.EncodeToString(randomBytes[:]) +} + +func writeSyncFile(filename string, data []byte, perm os.FileMode) error { + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = f.Write(data) + if err2 := f.Sync(); err2 != nil && err == nil { + err = err2 + } + if err2 := f.Close(); err2 != nil && err == nil { + err = err2 + } + return err +} + +func writeFile(filename string, data []byte, perm os.FileMode) error { + tempname := filename + ".tmp." + randomFileSuffix() + if err := writeSyncFile(tempname, data, perm); err != nil { + return fmt.Errorf("error writing %s: %w", filename, err) + } + if err := os.Rename(tempname, filename); err != nil { + os.Remove(tempname) + return fmt.Errorf("error writing %s: %w", filename, err) + } + return nil +} + +func writeTextFile(filename string, fileText string, perm os.FileMode) error { + return writeFile(filename, []byte(fileText), perm) +} + +func writeJSONFile(filename string, data any, perm os.FileMode) error { + fileBytes, err := json.Marshal(data) + if err != nil { + return err + } + fileBytes = append(fileBytes, '\n') + return writeFile(filename, fileBytes, perm) +} + +func fileExists(filename string) bool { + _, err := os.Lstat(filename) + return err == nil +} + +// countFiles returns the number of non-hidden files in dirPath whose +// name ends in suffix. Returns 0 if dirPath does not exist. +func countFiles(dirPath string, suffix string) (int, error) { + entries, err := os.ReadDir(dirPath) + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } else if err != nil { + return 0, err + } + count := 0 + for _, entry := range entries { + if !entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") && strings.HasSuffix(entry.Name(), suffix) { + count++ + } + } + return count, nil +} + +func tailFile(filename string, linesWanted int) ([]byte, int, error) { + file, err := os.Open(filename) + if err != nil { + return nil, 0, err + } + defer file.Close() + return tail(file, linesWanted, 4096) +} + +func tail(r io.ReadSeeker, linesWanted int, chunkSize int) ([]byte, int, error) { + var buf []byte + linesGot := 0 + + offset, err := r.Seek(0, io.SeekEnd) + if err != nil { + return nil, 0, err + } + for offset > 0 { + readSize := chunkSize + if offset < int64(readSize) { + readSize = int(offset) + } + offset -= int64(readSize) + if _, err := r.Seek(offset, io.SeekStart); err != nil { + return nil, 0, err + } + buf = slices.Grow(buf, readSize) + copy(buf[readSize:len(buf)+readSize], buf) + buf = buf[:len(buf)+readSize] + if _, err := io.ReadFull(r, buf[:readSize]); err != nil { + return nil, 0, err + } + for i := readSize; i > 0; i-- { + if buf[i-1] == '\n' { + if linesGot == linesWanted { + return buf[i:], linesGot, nil + } + linesGot++ + } + } + } + return buf, linesGot, nil +} diff --git a/third_party/certspotter/monitor/fsstate.go b/third_party/certspotter/monitor/fsstate.go new file mode 100644 index 0000000..3bb1f75 --- /dev/null +++ b/third_party/certspotter/monitor/fsstate.go @@ -0,0 +1,416 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" +) + +const keepErrorDays = 7 +const errorDateFormat = "2006-01-02" + +type FilesystemState struct { + StateDir string + CacheDir string + SaveCerts bool + Script string + ScriptDir string + Email []string + Stdout bool + errorMu sync.Mutex +} + +func (s *FilesystemState) logStateDir(logID LogID) string { + return filepath.Join(s.StateDir, "logs", logID.Base64URLString()) +} + +func (s *FilesystemState) Prepare(ctx context.Context) error { + if err := prepareStateDir(s.StateDir); err != nil { + return err + } + if err := prepareCacheDir(s.CacheDir); err != nil { + return err + } + return nil +} + +func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error { + var ( + stateDirPath = s.logStateDir(logID) + sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") + malformedDirPath = filepath.Join(stateDirPath, "malformed_entries") + healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks") + errorsDirPath = filepath.Join(stateDirPath, "errors") + ) + for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath, errorsDirPath} { + if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + return nil +} + +// ListLogs returns the IDs of the logs that have a state directory (i.e., for +// which PrepareLog has been called at some point). Note that LoadLogState can +// still return nil for a returned log ID, if no state has been stored yet. +func (s *FilesystemState) ListLogs(ctx context.Context) ([]LogID, error) { + entries, err := os.ReadDir(filepath.Join(s.StateDir, "logs")) + if errors.Is(err, fs.ErrNotExist) { + return []LogID{}, nil + } else if err != nil { + return nil, err + } + logIDs := make([]LogID, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + if strings.HasPrefix(name, ".") || !entry.IsDir() { + continue + } + nameBytes, err := base64.RawURLEncoding.DecodeString(name) + if err != nil || len(nameBytes) != 32 { + continue + } + var logID LogID + copy(logID[:], nameBytes) + logIDs = append(logIDs, logID) + } + return logIDs, nil +} + +func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) { + filePath := filepath.Join(s.logStateDir(logID), "state.json") + fileBytes, err := os.ReadFile(filePath) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, err + } + state := new(LogState) + if err := json.Unmarshal(fileBytes, state); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return state, nil +} + +func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error { + filePath := filepath.Join(s.logStateDir(logID), "state.json") + return writeJSONFile(filePath, state, 0666) +} + +func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) (*StoredSTH, error) { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return storeSTHInDir(sthsDirPath, sth) +} + +func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*StoredSTH, error) { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return loadSTHsFromDir(sthsDirPath) +} + +func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return removeSTHFromDir(sthsDirPath, sth) +} + +// LoadLargestSTH returns the stored STH with the largest tree size, breaking +// ties by later timestamp. Returns nil if no STHs are stored for the log. +func (s *FilesystemState) LoadLargestSTH(ctx context.Context, logID LogID) (*StoredSTH, error) { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return loadLargestSTHFromDir(sthsDirPath) +} + +// CountSTHs returns the number of STHs stored for the log. +func (s *FilesystemState) CountSTHs(ctx context.Context, logID LogID) (int, error) { + return countFiles(filepath.Join(s.logStateDir(logID), "unverified_sths"), ".json") +} + +// CountMalformedEntries returns the number of malformed entries recorded for the log. +func (s *FilesystemState) CountMalformedEntries(ctx context.Context, logID LogID) (int, error) { + return countFiles(filepath.Join(s.logStateDir(logID), "malformed_entries"), ".json") +} + +// CountHealthCheckFailures returns the number of health check failures recorded for the log. +func (s *FilesystemState) CountHealthCheckFailures(ctx context.Context, logID LogID) (int, error) { + return countFiles(filepath.Join(s.logStateDir(logID), "healthchecks"), ".txt") +} + +func (s *FilesystemState) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error { + filePath := filepath.Join(s.CacheDir, "issuers", hex.EncodeToString(fingerprint[:])) + return writeFile(filePath, issuer, 0666) +} + +func (s *FilesystemState) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) { + filePath := filepath.Join(s.CacheDir, "issuers", hex.EncodeToString(fingerprint[:])) + issuer, err := os.ReadFile(filePath) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, err + } else { + return issuer, err + } +} + +func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error { + var ( + tbsFingerprint = hex.EncodeToString(cert.TBSSHA256[:]) + certFingerprint = hex.EncodeToString(cert.SHA256[:]) + ) + var ( + tbsDir = filepath.Join(s.StateDir, "certs", tbsFingerprint[0:2]) + certDir = filepath.Join(s.StateDir, "certs", certFingerprint[0:2]) + ) + + notifiedPath := filepath.Join(tbsDir, "."+tbsFingerprint+".notified") + if fileExists(notifiedPath) { + return nil + } + // previous versions of certspotter used these files to record that a notification was sent + for _, filename := range []string{ + "." + certFingerprint + ".notified", + certFingerprint + ".cert.pem", + certFingerprint + ".precert.pem", + } { + if fileExists(filepath.Join(certDir, filename)) { + return nil + } + } + + var paths *certPaths + if s.SaveCerts { + if err := os.Mkdir(tbsDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err) + } + if err := os.Mkdir(certDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err) + } + paths = &certPaths{ + certPath: filepath.Join(certDir, certFingerprint+".pem"), + jsonPath: filepath.Join(certDir, certFingerprint+".v1.json"), + textPath: filepath.Join(certDir, certFingerprint+".txt"), + } + if err := writeCertFiles(cert, paths); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) + } + } else { + paths = &certPaths{ + temporary: true, + + certPath: filepath.Join(os.TempDir(), "certspotter-"+rand.Text()+".pem"), + jsonPath: filepath.Join(os.TempDir(), "certspotter-"+rand.Text()+".v1.json"), + textPath: filepath.Join(os.TempDir(), "certspotter-"+rand.Text()+".txt"), + } + defer func() { + os.Remove(paths.certPath) + os.Remove(paths.jsonPath) + os.Remove(paths.textPath) + }() + if err := writeCertFiles(cert, paths); err != nil { + return fmt.Errorf("error saving certificate %x to temporary files: %w", cert.SHA256, err) + } + } + + if err := s.notify(ctx, ¬ification{ + summary: certNotificationSummary(cert), + environ: certNotificationEnviron(cert, paths), + text: certNotificationText(cert, paths), + }); err != nil { + return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err) + } + + if s.SaveCerts { + if err := os.WriteFile(notifiedPath, nil, 0666); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) + } + } + + return nil +} + +func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error { + var ( + dirPath = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries") + entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index)) + textPath = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)) + ) + + summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.GetMonitoringURL()) + leafHash := merkletree.HashLeaf(entry.LeafInput()) + + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n") + writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.GetMonitoringURL())) + writeField("Leaf Hash", leafHash.Base64String()) + writeField("Error", parseError.Error()) + + if err := writeJSONFile(entryPath, entry.Entry, 0666); err != nil { + return fmt.Errorf("error saving JSON file: %w", err) + } + if err := writeTextFile(textPath, text.String(), 0666); err != nil { + return fmt.Errorf("error saving texT file: %w", err) + } + + environ := []string{ + "EVENT=malformed_cert", + "SUMMARY=" + summary, + "LOG_URI=" + entry.Log.GetMonitoringURL(), + "ENTRY_INDEX=" + fmt.Sprint(entry.Index), + "LEAF_HASH=" + leafHash.Base64String(), + "PARSE_ERROR=" + parseError.Error(), + "ENTRY_FILENAME=" + entryPath, + "TEXT_FILENAME=" + textPath, + "CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented + } + + if err := s.notify(ctx, ¬ification{ + environ: environ, + summary: summary, + text: text.String(), + }); err != nil { + return err + } + return nil +} + +func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string { + if ctlog == nil { + return filepath.Join(s.StateDir, "healthchecks") + } else { + return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks") + } +} + +func (s *FilesystemState) errorDir(ctlog *loglist.Log) string { + if ctlog == nil { + return filepath.Join(s.StateDir, "errors") + } + return filepath.Join(s.logStateDir(ctlog.LogID), "errors") +} + +func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error { + textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename()) + environ := []string{ + "EVENT=error", + "SUMMARY=" + info.Summary(), + "TEXT_FILENAME=" + textPath, + } + text := info.Text() + if err := writeTextFile(textPath, text, 0666); err != nil { + return fmt.Errorf("error saving text file: %w", err) + } + if err := s.notify(ctx, ¬ification{ + environ: environ, + summary: info.Summary(), + text: text, + }); err != nil { + return err + } + return nil +} + +func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error { + if ctlog == nil { + log.Print(notifyErr) + } + + var ( + now = time.Now() + filePath = filepath.Join(s.errorDir(ctlog), now.Format(errorDateFormat)) + line = now.Format(time.RFC3339) + " " + notifyErr.Error() + "\n" + ) + + s.errorMu.Lock() + defer s.errorMu.Unlock() + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return err + } + defer file.Close() + if _, err := file.WriteString(line); err != nil { + return err + } + return file.Close() +} + +func (s *FilesystemState) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) { + dir := s.errorDir(ctlog) + now := time.Now() + var buf []byte + for daysBack := 0; count > 0 && daysBack < keepErrorDays; daysBack++ { + datePath := filepath.Join(dir, now.AddDate(0, 0, -daysBack).Format(errorDateFormat)) + dateBuf, dateLines, err := tailFile(datePath, count) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return "", err + } + buf = append(dateBuf, buf...) + count -= dateLines + } + return string(buf), nil +} + +func (s *FilesystemState) PruneOldErrors() { + cutoff := time.Now().AddDate(0, 0, -keepErrorDays) + pruneDir := func(dir string) { + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return + } else if err != nil { + log.Printf("unable to read error directory: %s", err) + return + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + date, err := time.Parse(errorDateFormat, entry.Name()) + if err != nil { + continue + } + if date.Before(cutoff) { + if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Printf("unable to remove old error file: %s", err) + } + } + } + } + pruneDir(filepath.Join(s.StateDir, "errors")) + logsDir := filepath.Join(s.StateDir, "logs") + logDirs, err := os.ReadDir(logsDir) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Printf("unable to read logs directory: %s", err) + return + } + for _, d := range logDirs { + if !d.IsDir() { + continue + } + pruneDir(filepath.Join(logsDir, d.Name(), "errors")) + } +} diff --git a/third_party/certspotter/monitor/healthcheck.go b/third_party/certspotter/monitor/healthcheck.go new file mode 100644 index 0000000..c7772de --- /dev/null +++ b/third_party/certspotter/monitor/healthcheck.go @@ -0,0 +1,188 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "fmt" + "strings" + "time" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" +) + +const recentErrorCount = 10 + +func healthCheckFilename() string { + return time.Now().UTC().Format(time.RFC3339) + ".txt" +} + +func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error { + var ( + position uint64 + lastSuccess time.Time + verifiedSTH *cttypes.SignedTreeHead + ) + + if state, err := config.State.LoadLogState(ctx, ctlog.LogID); err != nil { + return fmt.Errorf("error loading log state: %w", err) + } else if state != nil { + if time.Since(state.LastSuccess) < config.HealthCheckInterval { + // log is healthy + return nil + } + + position = state.DownloadPosition.Size() + lastSuccess = state.LastSuccess + verifiedSTH = state.VerifiedSTH + } + + sths, err := config.State.LoadSTHs(ctx, ctlog.LogID) + if err != nil { + return fmt.Errorf("error loading STHs: %w", err) + } + + var errorsDir string + if fsstate, ok := config.State.(*FilesystemState); ok { + errorsDir = fsstate.errorDir(ctlog) + } + + if len(sths) == 0 { + errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount) + if err != nil { + return fmt.Errorf("error getting recent errors: %w", err) + } + info := &StaleSTHInfo{ + Log: ctlog, + LastSuccess: lastSuccess, + LatestSTH: verifiedSTH, + RecentErrors: errors, + ErrorsDir: errorsDir, + } + if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil { + return fmt.Errorf("error notifying about stale STH: %w", err) + } + } else { + errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount) + if err != nil { + return fmt.Errorf("error getting recent errors: %w", err) + } + info := &BacklogInfo{ + Log: ctlog, + LatestSTH: sths[len(sths)-1], + Position: position, + RecentErrors: errors, + ErrorsDir: errorsDir, + } + if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil { + return fmt.Errorf("error notifying about backlog: %w", err) + } + } + + return nil +} + +type HealthCheckFailure interface { + Summary() string + Text() string +} + +type StaleSTHInfo struct { + Log *loglist.Log + LastSuccess time.Time // may be zero + LatestSTH *cttypes.SignedTreeHead // may be nil + RecentErrors string + ErrorsDir string +} + +type BacklogInfo struct { + Log *loglist.Log + LatestSTH *StoredSTH + Position uint64 + RecentErrors string + ErrorsDir string +} + +type StaleLogListInfo struct { + Source string + LastSuccess time.Time + RecentErrors string + ErrorsDir string +} + +func (e *StaleSTHInfo) LastSuccessString() string { + if e.LastSuccess.IsZero() { + return "never" + } else { + return e.LastSuccess.String() + } +} +func (e *BacklogInfo) Backlog() uint64 { + return e.LatestSTH.TreeSize - e.Position +} + +func (e *StaleSTHInfo) Summary() string { + return fmt.Sprintf("Unable to contact %s since %s", e.Log.GetMonitoringURL(), e.LastSuccessString()) +} +func (e *BacklogInfo) Summary() string { + return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.GetMonitoringURL()) +} +func (e *StaleLogListInfo) Summary() string { + return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess) +} + +func (e *StaleSTHInfo) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL(), e.LastSuccessString()) + fmt.Fprintf(text, "\n") + if e.LatestSTH != nil { + fmt.Fprintf(text, "Latest known log size = %d\n", e.LatestSTH.TreeSize) + } else { + fmt.Fprintf(text, "Latest known log size = none\n") + } + if e.RecentErrors != "" { + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir) + fmt.Fprintf(text, "\n") + fmt.Fprint(text, e.RecentErrors) + } + return text.String() +} +func (e *BacklogInfo) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL()) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.StoredAt) + fmt.Fprintf(text, "Current position = %d\n", e.Position) + fmt.Fprintf(text, " Backlog = %d\n", e.Backlog()) + if e.RecentErrors != "" { + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir) + fmt.Fprintf(text, "\n") + fmt.Fprint(text, e.RecentErrors) + } + return text.String() +} +func (e *StaleLogListInfo) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n") + if e.RecentErrors != "" { + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir) + fmt.Fprintf(text, "\n") + fmt.Fprint(text, e.RecentErrors) + } + return text.String() +} + +// TODO-3: make the errors more actionable diff --git a/third_party/certspotter/monitor/loglist.go b/third_party/certspotter/monitor/loglist.go new file mode 100644 index 0000000..3d88290 --- /dev/null +++ b/third_party/certspotter/monitor/loglist.go @@ -0,0 +1,45 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "fmt" + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" +) + +type LogID = cttypes.LogID + +func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) { + list, newToken, err := loglist.LoadIfModified(ctx, source, token) + if err != nil { + return nil, nil, err + } + + logs := make(map[LogID]*loglist.Log) + for operatorIndex := range list.Operators { + for logIndex := range list.Operators[operatorIndex].Logs { + log := &list.Operators[operatorIndex].Logs[logIndex] + if _, exists := logs[log.LogID]; exists { + return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String()) + } + logs[log.LogID] = log + } + for logIndex := range list.Operators[operatorIndex].TiledLogs { + log := &list.Operators[operatorIndex].TiledLogs[logIndex] + if _, exists := logs[log.LogID]; exists { + return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String()) + } + logs[log.LogID] = log + } + } + return logs, newToken, nil +} diff --git a/third_party/certspotter/monitor/mailutils.go b/third_party/certspotter/monitor/mailutils.go new file mode 100644 index 0000000..ea06d4f --- /dev/null +++ b/third_party/certspotter/monitor/mailutils.go @@ -0,0 +1,34 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "crypto/rand" + "encoding/hex" + "os" +) + +const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700" + +func generateMessageID() string { + var randomBytes [16]byte + if _, err := rand.Read(randomBytes[:]); err != nil { + panic(err) + } + return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org" +} + +func sendmailPath() string { + if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" { + return envVar + } else { + return "/usr/sbin/sendmail" + } +} diff --git a/third_party/certspotter/monitor/monitor.go b/third_party/certspotter/monitor/monitor.go new file mode 100644 index 0000000..8281332 --- /dev/null +++ b/third_party/certspotter/monitor/monitor.go @@ -0,0 +1,608 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "errors" + "fmt" + "golang.org/x/sync/errgroup" + "golang.org/x/time/rate" + "log" + mathrand "math/rand/v2" + "net/url" + "slices" + "time" + + "software.sslmate.com/src/certspotter/ctclient" + "software.sslmate.com/src/certspotter/ctcrypto" + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" + "software.sslmate.com/src/certspotter/sequencer" +) + +const ( + getSTHInterval = 5 * time.Minute + maxPartialTileAge = 5 * time.Minute +) + +func downloadJobSize(ctlog *loglist.Log) uint64 { + if ctlog.IsStaticCTAPI() { + return ctclient.StaticTileWidth + } else if ctlog.CertspotterDownloadSize != 0 { + return uint64(ctlog.CertspotterDownloadSize) + } else { + return 1000 + } +} + +func downloadWorkers(ctlog *loglist.Log) int { + if ctlog.CertspotterDownloadQPS != 0 { + // parallelism is effectively governed by the rate limit so for now just hard code a number here + return 10 + } else if ctlog.CertspotterDownloadJobs != 0 { + return ctlog.CertspotterDownloadJobs + } else { + return 1 + } +} + +func downloadRateLimit(ctlog *loglist.Log) rate.Limit { + if ctlog.CertspotterDownloadQPS != 0 { + return rate.Limit(ctlog.CertspotterDownloadQPS) + } else { + return rate.Inf + } +} + +type verifyEntriesError struct { + sth *cttypes.SignedTreeHead + entriesRootHash merkletree.Hash +} + +func (e *verifyEntriesError) Error() string { + return fmt.Sprintf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", e.sth.TreeSize, e.sth.RootHash, e.entriesRootHash) +} + +type partialError struct { + error +} + +func (e partialError) Unwrap() error { + return e.error +} + +func isPartialError(err error) bool { + var pe partialError + return errors.As(err, &pe) +} + +func withRetry(ctx context.Context, config *Config, ctlog *loglist.Log, maxRetries int, f func() error) error { + minSleep := 100 * time.Millisecond + numRetries := 0 + for ctx.Err() == nil { + err := f() + if err == nil || errors.Is(err, context.Canceled) { + return err + } + if numRetries > 0 { + err = fmt.Errorf("%w (retried %d times)", err, numRetries) + } + if isPartialError(err) { + recordError(ctx, config, ctlog, err) + return nil + } + if maxRetries != -1 && numRetries >= maxRetries { + return err + } + recordError(ctx, config, ctlog, err) + sleepTime := minSleep + mathrand.N(minSleep) + if err := sleep(ctx, sleepTime); err != nil { + return err + } + minSleep = min(minSleep*2, 30*time.Second) + numRetries++ + } + return ctx.Err() +} + +func getEntriesFull(ctx context.Context, client ctclient.Log, startInclusive, endExclusive uint64) ([]ctclient.Entry, error) { + allEntries := make([]ctclient.Entry, 0, endExclusive-startInclusive) + for startInclusive < endExclusive { + entries, err := client.GetEntries(ctx, startInclusive, endExclusive-1) + if err != nil { + return nil, err + } + allEntries = append(allEntries, entries...) + startInclusive += uint64(len(entries)) + } + return allEntries, nil +} + +func getAuthenticSTH(ctx context.Context, ctlog *loglist.Log, client ctclient.Log) (*cttypes.SignedTreeHead, string, error) { + sth, url, err := client.GetSTH(ctx) + if err != nil { + return nil, "", err + } + if err := ctcrypto.PublicKey(ctlog.Key).Verify(ctcrypto.SignatureInputForSTH(sth), sth.Signature); err != nil { + return nil, "", fmt.Errorf("STH has invalid signature: %w", err) + } + return sth, url, nil +} + +type logClient struct { + config *Config + log *loglist.Log + client ctclient.Log + lim *rate.Limiter +} + +func (client *logClient) GetSTH(ctx context.Context) (sth *cttypes.SignedTreeHead, url string, err error) { + err = withRetry(ctx, client.config, client.log, -1, func() error { + if err := client.lim.Wait(ctx); err != nil { + return err + } + sth, url, err = getAuthenticSTH(ctx, client.log, client.client) + return err + }) + return +} +func (client *logClient) GetRoots(ctx context.Context) (roots [][]byte, err error) { + err = withRetry(ctx, client.config, client.log, -1, func() error { + if err := client.lim.Wait(ctx); err != nil { + return err + } + roots, err = client.client.GetRoots(ctx) + return err + }) + return +} +func (client *logClient) GetEntries(ctx context.Context, startInclusive, endInclusive uint64) (entries []ctclient.Entry, err error) { + err = withRetry(ctx, client.config, client.log, -1, func() error { + if err := client.lim.Wait(ctx); err != nil { + return err + } + entries, err = client.client.GetEntries(ctx, startInclusive, endInclusive) + if err != nil && len(entries) != 0 { + err = partialError{err} + } + return err + }) + return +} +func (client *logClient) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (tree *merkletree.CollapsedTree, err error) { + err = withRetry(ctx, client.config, client.log, -1, func() error { + if err := client.lim.Wait(ctx); err != nil { + return err + } + tree, err = client.client.ReconstructTree(ctx, sth) + return err + }) + return +} + +type issuerGetter struct { + config *Config + log *loglist.Log + logGetter ctclient.IssuerGetter +} + +func (ig *issuerGetter) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) { + if issuer, err := ig.config.State.LoadIssuer(ctx, fingerprint); err != nil { + log.Printf("error loading cached issuer %x (issuer will be retrieved from log instead): %s", *fingerprint, err) + } else if issuer != nil { + return issuer, nil + } + + var issuer []byte + if err := withRetry(ctx, ig.config, ig.log, 7, func() error { + var err error + issuer, err = ig.logGetter.GetIssuer(ctx, fingerprint) + return err + }); err != nil { + return nil, err + } + + if err := ig.config.State.StoreIssuer(ctx, fingerprint, issuer); err != nil { + log.Printf("error caching issuer %x (issuer will be re-retrieved from log in the future): %s", *fingerprint, err) + } + + return issuer, nil +} + +func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.IssuerGetter, error) { + switch { + case ctlog.IsRFC6962(): + logURL, err := url.Parse(ctlog.URL) + if err != nil { + return nil, nil, fmt.Errorf("log has invalid URL: %w", err) + } + return &logClient{ + config: config, + log: ctlog, + client: &ctclient.RFC6962Log{URL: logURL}, + lim: rate.NewLimiter(downloadRateLimit(ctlog), 1), + }, nil, nil + case ctlog.IsStaticCTAPI(): + submissionURL, err := url.Parse(ctlog.SubmissionURL) + if err != nil { + return nil, nil, fmt.Errorf("log has invalid submission URL: %w", err) + } + monitoringURL, err := url.Parse(ctlog.MonitoringURL) + if err != nil { + return nil, nil, fmt.Errorf("log has invalid monitoring URL: %w", err) + } + client := &ctclient.StaticLog{ + SubmissionURL: submissionURL, + MonitoringURL: monitoringURL, + ID: ctlog.LogID, + } + return &logClient{ + config: config, + log: ctlog, + client: client, + lim: rate.NewLimiter(downloadRateLimit(ctlog), 1), + }, &issuerGetter{ + config: config, + log: ctlog, + logGetter: client, + }, nil + default: + return nil, nil, errors.New("log uses unknown protocol") + } +} + +func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) (returnedErr error) { + client, issuerGetter, err := newLogClient(config, ctlog) + if err != nil { + return err + } + if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil { + return fmt.Errorf("error preparing state: %w", err) + } + state, err := config.State.LoadLogState(ctx, ctlog.LogID) + if err != nil { + return fmt.Errorf("error loading log state: %w", err) + } + if state == nil { + if config.StartAtEnd { + sth, _, err := client.GetSTH(ctx) + if err != nil { + return err + } + tree, err := client.ReconstructTree(ctx, sth) + if err != nil { + return err + } + state = &LogState{ + DownloadPosition: tree, + VerifiedPosition: tree, + VerifiedSTH: sth, + LastSuccess: time.Now(), + } + } else { + state = &LogState{ + DownloadPosition: merkletree.EmptyCollapsedTree(), + VerifiedPosition: merkletree.EmptyCollapsedTree(), + VerifiedSTH: nil, + LastSuccess: time.Now(), + } + } + if config.Verbose { + log.Printf("%s: monitoring brand new log starting from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size()) + } + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) + } + } else { + if config.Verbose { + log.Printf("%s: resuming monitoring from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size()) + } + } + + defer func() { + storeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := config.State.StoreLogState(storeCtx, ctlog.LogID, state); err != nil && returnedErr == nil { + returnedErr = fmt.Errorf("error storing log state: %w", err) + } + }() + +retry: + position := state.DownloadPosition.Size() + + // logs are monitored using the following pipeline of workers, with each worker sending results to the next worker: + // 1 getSTHWorker ==> 1 generateBatchesWorker ==> multiple downloadWorkers ==> multiple processWorkers ==> 1 saveStateWorker + // getSTHWorker - periodically download STHs from the log + // generateBatchesWorker - generate batches of work + // downloadWorkers - download the entries in each batch + // processWorkers - process the certificates (store/notify if matches watch list) in each batch + // saveStateWorker - builds the Merkle Tree and compares against STHs + + sths := make(chan *cttypes.SignedTreeHead, 1) + batches := make(chan *batch, downloadWorkers(ctlog)) + processedBatches := sequencer.New[batch](0, uint64(downloadWorkers(ctlog))*10) + + group, gctx := errgroup.WithContext(ctx) + group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client, sths) }) + group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, sths, batches) }) + for range downloadWorkers(ctlog) { + downloadedBatches := make(chan *batch, 1) + group.Go(func() error { return downloadWorker(gctx, config, ctlog, client, batches, downloadedBatches) }) + group.Go(func() error { + return processWorker(gctx, config, ctlog, issuerGetter, downloadedBatches, processedBatches) + }) + } + group.Go(func() error { return saveStateWorker(gctx, config, ctlog, state, processedBatches) }) + + err = group.Wait() + if verifyErr := (*verifyEntriesError)(nil); errors.As(err, &verifyErr) { + recordError(ctx, config, ctlog, verifyErr) + state.rewindDownloadPosition() + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) + } + if err := sleep(ctx, 5*time.Minute); err != nil { + return err + } + goto retry + } + return err +} + +func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, sthsOut chan<- *cttypes.SignedTreeHead) error { + ticker := time.NewTicker(getSTHInterval) + defer ticker.Stop() + for { + sth, _, err := client.GetSTH(ctx) + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case sthsOut <- sth: + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +type batch struct { + number uint64 + begin, end uint64 + discoveredAt time.Time // time at which we became aware of the log having entries in range [begin,end) + sths []*StoredSTH // STHs with sizes in range [begin,end], sorted by TreeSize + entries []ctclient.Entry // in range [begin,end) +} + +// Create a batch starting from begin, based on sths (which must be non-empty, sorted by TreeSize, and contain only STHs with TreeSize >= begin). Returns the batch, plus the remaining STHs. +func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize uint64) (*batch, []*StoredSTH) { + batch := &batch{ + number: number, + begin: begin, + discoveredAt: sths[0].StoredAt, + } + maxEnd := (begin/downloadJobSize + 1) * downloadJobSize + for _, sth := range sths { + if sth.StoredAt.Before(batch.discoveredAt) { + batch.discoveredAt = sth.StoredAt + } + if sth.TreeSize <= maxEnd { + batch.end = sth.TreeSize + batch.sths = append(batch.sths, sth) + } else { + batch.end = maxEnd + break + } + } + return batch, sths[len(batch.sths):] +} + +// insert sth into sths, which is sorted by TreeSize, and return a new, still-sorted slice. +// if an equivalent STH is already in sths, it is returned unchanged. +func insertSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH { + i := len(sths) + for i > 0 { + if sths[i-1].Same(&sth.SignedTreeHead) { + return sths + } + if sths[i-1].TreeSize < sth.TreeSize { + break + } + i-- + } + return slices.Insert(sths, i, sth) +} + +func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, sthsIn <-chan *cttypes.SignedTreeHead, batchesOut chan<- *batch) error { + downloadJobSize := downloadJobSize(ctlog) + + sths, err := config.State.LoadSTHs(ctx, ctlog.LogID) + if err != nil { + return fmt.Errorf("error loading STHs: %w", err) + } + // sths is sorted by TreeSize but may contain STHs with TreeSize < position; get rid of them + for len(sths) > 0 && sths[0].TreeSize < position { + // TODO-4: audit sths[0] against log's verified STH + if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sths[0].SignedTreeHead); err != nil { + return fmt.Errorf("error removing STH: %w", err) + } + sths = sths[1:] + } + // from this point, sths is sorted by TreeSize and contains only STHs with TreeSize >= position + handleSTH := func(sth *cttypes.SignedTreeHead) error { + if sth.TreeSize < position { + // TODO-4: audit against log's verified STH + } else { + storedSTH, err := config.State.StoreSTH(ctx, ctlog.LogID, sth) + if err != nil { + return fmt.Errorf("error storing STH: %w", err) + } + sths = insertSTH(sths, storedSTH) + } + return nil + } + + var number uint64 + for { + for len(sths) == 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case sth := <-sthsIn: + if err := handleSTH(sth); err != nil { + return err + } + } + } + + batch, remainingSTHs := newBatch(number, position, sths, downloadJobSize) + + if ctlog.IsStaticCTAPI() && batch.end%downloadJobSize != 0 { + // Wait to download this partial tile until it's old enough + if age := time.Since(batch.discoveredAt); age < maxPartialTileAge { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(maxPartialTileAge - age): + case sth := <-sthsIn: + if err := handleSTH(sth); err != nil { + return err + } + continue + } + } + } + + select { + case <-ctx.Done(): + return ctx.Err() + case sth := <-sthsIn: + if err := handleSTH(sth); err != nil { + return err + } + case batchesOut <- batch: + number = batch.number + 1 + position = batch.end + sths = remainingSTHs + } + } +} + +func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, batchesIn <-chan *batch, batchesOut chan<- *batch) error { + for { + var batch *batch + select { + case <-ctx.Done(): + return ctx.Err() + case batch = <-batchesIn: + } + + entries, err := getEntriesFull(ctx, client, batch.begin, batch.end) + if err != nil { + return err + } + batch.entries = entries + + select { + case <-ctx.Done(): + return ctx.Err() + case batchesOut <- batch: + } + } +} + +func processWorker(ctx context.Context, config *Config, ctlog *loglist.Log, issuerGetter ctclient.IssuerGetter, batchesIn <-chan *batch, batchesOut *sequencer.Channel[batch]) error { + for { + var batch *batch + select { + case <-ctx.Done(): + return ctx.Err() + case batch = <-batchesIn: + } + for offset, entry := range batch.entries { + index := batch.begin + uint64(offset) + if err := processLogEntry(ctx, config, issuerGetter, &LogEntry{ + Entry: entry, + Index: index, + Log: ctlog, + }); err != nil { + return fmt.Errorf("error processing entry %d: %w", index, err) + } + } + if err := batchesOut.Add(ctx, batch.number, batch); err != nil { + return err + } + } +} + +func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, state *LogState, batchesIn *sequencer.Channel[batch]) error { + for { + batch, err := batchesIn.Next(ctx) + if err != nil { + return err + } + if batch.begin != state.DownloadPosition.Size() { + panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin)) + } + for { + for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() { + sth := batch.sths[0] + batch.sths = batch.sths[1:] + if rootHash := state.DownloadPosition.CalculateRoot(); sth.RootHash != rootHash { + return &verifyEntriesError{ + sth: &sth.SignedTreeHead, + entriesRootHash: rootHash, + } + } + state.advanceVerifiedPosition() + state.LastSuccess = sth.StoredAt + state.VerifiedSTH = &sth.SignedTreeHead + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) + } + // don't remove the STH until state has been durably stored + if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sth.SignedTreeHead); err != nil { + return fmt.Errorf("error removing verified STH: %w", err) + } + if config.Verbose { + log.Printf("%s: verified position is now %d", ctlog.GetMonitoringURL(), sth.SignedTreeHead.TreeSize) + } + } + if len(batch.entries) == 0 { + break + } + entry := batch.entries[0] + batch.entries = batch.entries[1:] + leafHash := merkletree.HashLeaf(entry.LeafInput()) + state.DownloadPosition.Add(leafHash) + } + + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) + } + } +} + +func sleep(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/third_party/certspotter/monitor/notify.go b/third_party/certspotter/monitor/notify.go new file mode 100644 index 0000000..8ff5a2c --- /dev/null +++ b/third_party/certspotter/monitor/notify.go @@ -0,0 +1,166 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +var stdoutMu sync.Mutex + +type notification struct { + environ []string + summary string + text string +} + +func (s *FilesystemState) notify(ctx context.Context, notif *notification) error { + if s.Stdout { + writeToStdout(notif) + } + + if len(s.Email) > 0 { + if err := sendEmail(ctx, s.Email, notif); err != nil { + return err + } + } + + if s.Script != "" { + if err := execScript(ctx, s.Script, notif); err != nil { + return err + } + } + + if s.ScriptDir != "" { + if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil { + return err + } + } + + return nil +} + +func writeToStdout(notif *notification) { + stdoutMu.Lock() + defer stdoutMu.Unlock() + os.Stdout.WriteString(notif.text + "\n") +} + +func sendEmail(ctx context.Context, to []string, notif *notification) error { + stdin := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + from := os.Getenv("EMAIL") + + if from != "" { + fmt.Fprintf(stdin, "From: %s\n", from) + } + fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", ")) + fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary) + fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat)) + fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID()) + fmt.Fprintf(stdin, "Mime-Version: 1.0\n") + fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n") + fmt.Fprintf(stdin, "X-Mailer: certspotter\n") + fmt.Fprintf(stdin, "\n") + fmt.Fprint(stdin, notif.text) + + args := []string{"-i"} + if from != "" { + args = append(args, "-f", from) + } + args = append(args, "--") + args = append(args, to...) + + sendmailCtx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Minute)) + defer cancel() + sendmail := exec.CommandContext(sendmailCtx, sendmailPath(), args...) + sendmail.Stdin = stdin + sendmail.Stderr = stderr + sendmail.WaitDelay = 5 * time.Second + + if err := sendmail.Run(); err == nil || err == exec.ErrWaitDelay { + return nil + } else if sendmailCtx.Err() != nil && ctx.Err() == nil { + return fmt.Errorf("error sending email to %v: sendmail command timed out", to) + } else if ctx.Err() != nil { + // if the context was canceled, we can't be sure that the error is the fault of sendmail, so ignore it + return ctx.Err() + } else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() { + return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String())) + } else if isExitError { + return fmt.Errorf("error sending email to %v: sendmail terminated by signal with error %q", to, strings.TrimSpace(stderr.String())) + } else { + return fmt.Errorf("error sending email to %v: error running sendmail command: %w", to, err) + } +} + +func execScript(ctx context.Context, scriptName string, notif *notification) error { + stderr := new(bytes.Buffer) + + cmd := exec.CommandContext(ctx, scriptName) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, notif.environ...) + cmd.Stderr = stderr + cmd.WaitDelay = 5 * time.Second + + if err := cmd.Run(); err == nil || err == exec.ErrWaitDelay { + return nil + } else if ctx.Err() != nil { + // if the context was canceled, we can't be sure that the error is the fault of the script, so ignore it + return ctx.Err() + } else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() { + return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String())) + } else if isExitError { + return fmt.Errorf("script %q terminated by signal with error %q", scriptName, strings.TrimSpace(stderr.String())) + } else { + return fmt.Errorf("error executing script: %w", err) + } +} + +func execScriptDir(ctx context.Context, dirPath string, notif *notification) error { + dirents, err := os.ReadDir(dirPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err) + } + for _, dirent := range dirents { + if strings.HasPrefix(dirent.Name(), ".") { + continue + } + scriptPath := filepath.Join(dirPath, dirent.Name()) + info, err := os.Stat(scriptPath) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err) + } else if info.Mode().IsRegular() && isExecutable(info.Mode()) { + if err := execScript(ctx, scriptPath, notif); err != nil { + return err + } + } + } + return nil +} + +func isExecutable(mode os.FileMode) bool { + return mode&0111 != 0 +} diff --git a/third_party/certspotter/monitor/process.go b/third_party/certspotter/monitor/process.go new file mode 100644 index 0000000..958a686 --- /dev/null +++ b/third_party/certspotter/monitor/process.go @@ -0,0 +1,151 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + + "software.sslmate.com/src/certspotter" + "software.sslmate.com/src/certspotter/ctclient" + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" +) + +type LogEntry struct { + ctclient.Entry + Index uint64 + Log *loglist.Log +} + +func processLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry) error { + leaf, err := cttypes.ParseLeafInput(entry.LeafInput()) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err)) + } + switch leaf.TimestampedEntry.EntryType { + case cttypes.X509EntryType: + return processX509LogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryASN1Cert) + case cttypes.PrecertEntryType: + return processPrecertLogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryPreCert) + default: + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType)) + } +} + +func processX509LogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, cert *cttypes.ASN1Cert) error { + certInfo, err := certspotter.MakeCertInfoFromRawCert(*cert) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err)) + } + if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil { + certInfo.TBS = precertTBS + } else { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err)) + } + + getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) { + var ( + chain = []cttypes.ASN1Cert{*cert} + errs = []error{} + ) + if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil { + chain = append(chain, issuers...) + } else { + errs = append(errs, err) + } + return chain, errors.Join(errs...) + } + return processCertificate(ctx, config, entry, certInfo, getChain) +} + +func processPrecertLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, precert *cttypes.PreCert) error { + certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err)) + } + precertBytes, err := entry.Precertificate() + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error getting precert entry's precertificate: %w", err)) + } + + getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) { + var ( + chain = []cttypes.ASN1Cert{precertBytes} + errs = []error{} + ) + if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil { + chain = append(chain, issuers...) + } else { + errs = append(errs, err) + } + if _, err := certspotter.ValidatePrecert(precertBytes, precert.TBSCertificate); err != nil { + errs = append(errs, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err)) + } + return chain, errors.Join(errs...) + } + return processCertificate(ctx, config, entry, certInfo, getChain) +} + +func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, getChain func(context.Context) ([]cttypes.ASN1Cert, error)) error { + identifiers, err := certInfo.ParseIdentifiers() + if err != nil { + return processMalformedLogEntry(ctx, config, entry, err) + } + // CTCLOUD PATCH: hand identifiers directly to the in-process sink and skip + // the watchlist/chain/notification path. See CTCLOUD_PATCH.md. + if config.IdentifierSink != nil { + config.IdentifierSink.HandleIdentifiers(ctx, identifiers) + return nil + } + matched, watchItem := config.WatchList.Matches(identifiers) + if !matched { + return nil + } + + chain, chainErr := getChain(ctx) + if chainErr != nil { + if ctx.Err() != nil { + // Getting chain failed, but it was probably because our context + // has been canceled, so just act like we never called getChain. + return ctx.Err() + } + // Although getting the chain failed, we still want to notify + // the user about the matching certificate. We'll include chainErr in the + // notification so the user knows why the chain is missing or incorrect. + } + + cert := &DiscoveredCert{ + WatchItem: watchItem, + LogEntry: entry, + Info: certInfo, + Chain: chain, + ChainError: chainErr, + TBSSHA256: sha256.Sum256(certInfo.TBS.Raw), + SHA256: sha256.Sum256(chain[0]), + PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes), + Identifiers: identifiers, + } + + if err := config.State.NotifyCert(ctx, cert); err != nil { + return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err) + } + + return nil +} + +func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error { + if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil { + return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.GetMonitoringURL(), parseError, err) + } + return nil +} diff --git a/third_party/certspotter/monitor/state.go b/third_party/certspotter/monitor/state.go new file mode 100644 index 0000000..56ddc20 --- /dev/null +++ b/third_party/certspotter/monitor/state.go @@ -0,0 +1,91 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "time" + + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" +) + +type LogState struct { + DownloadPosition *merkletree.CollapsedTree `json:"download_position"` + VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"` + VerifiedSTH *cttypes.SignedTreeHead `json:"verified_sth"` + LastSuccess time.Time `json:"last_success"` +} + +func (state *LogState) rewindDownloadPosition() { + position := state.VerifiedPosition.Clone() + state.DownloadPosition = &position +} + +func (state *LogState) advanceVerifiedPosition() { + position := state.DownloadPosition.Clone() + state.VerifiedPosition = &position +} + +// Methods are safe to call concurrently. +type StateProvider interface { + // Initialize the state. Called before any other method in this interface. + // Idempotent: returns nil if the state is already initialized. + Prepare(context.Context) error + + // Initialize the state for the given log. Called before any other method + // with the log ID. Idempotent: returns nil if log state already initialized. + PrepareLog(context.Context, LogID) error + + // Store log state for retrieval by LoadLogState. + StoreLogState(context.Context, LogID, *LogState) error + + // Load log state that was previously stored with StoreLogState. + // Returns nil, nil if StoreLogState has not been called yet for this log. + LoadLogState(context.Context, LogID) (*LogState, error) + + // Store STH for retrieval by LoadSTHs. If an STH with the same + // timestamp and root hash is already stored, this STH can be ignored. + StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) (*StoredSTH, error) + + // Load all STHs for this log previously stored with StoreSTH. + // The returned slice must be sorted by tree size. + LoadSTHs(context.Context, LogID) ([]*StoredSTH, error) + + // Remove an STH so it is no longer returned by LoadSTHs. + RemoveSTH(context.Context, LogID, *cttypes.SignedTreeHead) error + + // Store a DER-encoded issuer certificate with the given fingerprint for + // retrieval by LoadIssuer. Returns nil if the issuer has already been stored. + StoreIssuer(context.Context, *[32]byte, []byte) error + + // Retrieve a DER-encoded issuer certificate previously stored with StoreIssuer. + // Returns nil, nil if this issuer certificate has not been stored. + LoadIssuer(context.Context, *[32]byte) ([]byte, error) + + // Called when a certificate matching the watch list is discovered. + NotifyCert(context.Context, *DiscoveredCert) error + + // Called when certspotter fails to parse a log entry. + NotifyMalformedEntry(context.Context, *LogEntry, error) error + + // Called when a health check fails. The log is nil if the + // feailure is not associated with a log. + NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error + + // Called when a non-fatal error occurs. The log is nil if the error is + // not associated with a log. Note that most errors are transient, and + // certspotter will retry the failed operation later. + NotifyError(context.Context, *loglist.Log, error) error + + // Retrieve the specified number of most recent errors. + GetErrors(context.Context, *loglist.Log, int) (string, error) +} diff --git a/third_party/certspotter/monitor/statedir.go b/third_party/certspotter/monitor/statedir.go new file mode 100644 index 0000000..d8d4198 --- /dev/null +++ b/third_party/certspotter/monitor/statedir.go @@ -0,0 +1,167 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter/cttypes" + "software.sslmate.com/src/certspotter/merkletree" + "strconv" + "strings" + "time" +) + +func readVersion(stateDir string) (int, error) { + path := filepath.Join(stateDir, "version") + + fileBytes, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + if fileExists(filepath.Join(stateDir, "evidence")) { + return 0, nil + } else { + return -1, nil + } + } else if err != nil { + return -1, err + } + + version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes))) + if err != nil { + return -1, fmt.Errorf("version file %q is malformed: %w", path, err) + } + + return version, nil +} + +func writeVersion(stateDir string) error { + return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666) +} + +func migrateLogStateDirV1(dir string) error { + var sth cttypes.SignedTreeHead + var tree merkletree.CollapsedTree + + sthPath := filepath.Join(dir, "sth.json") + sthData, err := os.ReadFile(sthPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + treePath := filepath.Join(dir, "tree.json") + treeData, err := os.ReadFile(treePath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + if err := json.Unmarshal(sthData, &sth); err != nil { + return fmt.Errorf("error unmarshaling %s: %w", sthPath, err) + } + if err := json.Unmarshal(treeData, &tree); err != nil { + return fmt.Errorf("error unmarshaling %s: %w", treePath, err) + } + + stateFile := LogState{ + DownloadPosition: &tree, + VerifiedPosition: &tree, + VerifiedSTH: &sth, + LastSuccess: time.Now(), + } + if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil { + return err + } + + if err := os.Remove(sthPath); err != nil { + return err + } + if err := os.Remove(treePath); err != nil { + return err + } + return nil +} + +func migrateStateDirV1(stateDir string) error { + if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) { + return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile) + } + + if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil { + for _, logDir := range logDirs { + if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() { + continue + } + if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil { + return fmt.Errorf("error migrating log state: %w", err) + } + } + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } + + if err := writeVersion(stateDir); err != nil { + return err + } + + if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + + return nil +} + +func prepareStateDir(stateDir string) error { + if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + + if version, err := readVersion(stateDir); err != nil { + return err + } else if version == -1 { + if err := writeVersion(stateDir); err != nil { + return err + } + } else if version == 0 { + return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir) + } else if version == 1 { + if err := migrateStateDirV1(stateDir); err != nil { + return err + } + } else if version > 2 { + return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir) + } + + for _, subdir := range []string{"certs", "logs", "healthchecks", "errors"} { + if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + + return nil +} + +func prepareCacheDir(cacheDir string) error { + if err := os.MkdirAll(cacheDir, 0777); err != nil { + return err + } + for _, subdir := range []string{"issuers"} { + if err := os.Mkdir(filepath.Join(cacheDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + return nil +} diff --git a/third_party/certspotter/monitor/sthdir.go b/third_party/certspotter/monitor/sthdir.go new file mode 100644 index 0000000..89aa38d --- /dev/null +++ b/third_party/certspotter/monitor/sthdir.go @@ -0,0 +1,178 @@ +// Copyright (C) 2017, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "cmp" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "software.sslmate.com/src/certspotter/cttypes" + "strconv" + "strings" + "time" +) + +type StoredSTH struct { + cttypes.SignedTreeHead + StoredAt time.Time `json:"stored_at"` // time at which the STH was first stored +} + +func loadSTHsFromDir(dirPath string) ([]*StoredSTH, error) { + entries, err := os.ReadDir(dirPath) + if errors.Is(err, fs.ErrNotExist) { + return []*StoredSTH{}, nil + } else if err != nil { + return nil, err + } + sths := make([]*StoredSTH, 0, len(entries)) + for _, entry := range entries { + filename := entry.Name() + if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") { + continue + } + sth, err := readSTHFile(filepath.Join(dirPath, filename)) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return nil, err + } + sths = append(sths, sth) + } + slices.SortFunc(sths, func(a, b *StoredSTH) int { return cmp.Compare(a.TreeSize, b.TreeSize) }) + return sths, nil +} + +// loadLargestSTHFromDir returns the STH in dirPath with the largest tree size, breaking +// ties by later timestamp. +func loadLargestSTHFromDir(dirPath string) (*StoredSTH, error) { + compareSTH := func(a, b *StoredSTH) int { + return cmp.Or( + cmp.Compare(a.TreeSize, b.TreeSize), + cmp.Compare(a.Timestamp, b.Timestamp), + ) + } + + entries, err := os.ReadDir(dirPath) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, err + } + + var largestSTH *StoredSTH + for _, entry := range entries { + filename := entry.Name() + if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") { + continue + } + if largestSTH != nil { + if treeSize, ok := sthFilenameTreeSize(filename); ok && treeSize < largestSTH.TreeSize { + continue + } + } + sth, err := readSTHFile(filepath.Join(dirPath, filename)) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return nil, err + } + if largestSTH == nil || compareSTH(sth, largestSTH) > 0 { + largestSTH = sth + } + } + return largestSTH, nil +} + +func readSTHFile(filePath string) (*StoredSTH, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return nil, err + } + sth := &StoredSTH{ + StoredAt: info.ModTime(), + } + + fileBytes, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("error reading %s: %w", filePath, err) + } + if err := json.Unmarshal(fileBytes, &sth.SignedTreeHead); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return sth, nil +} + +func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) (*StoredSTH, error) { + filePath := filepath.Join(dirPath, sthFilename(sth)) + + if info, err := os.Lstat(filePath); err == nil { + return &StoredSTH{ + SignedTreeHead: *sth, + StoredAt: info.ModTime(), + }, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + + if err := writeJSONFile(filePath, sth, 0666); err != nil { + return nil, err + } + + return &StoredSTH{ + SignedTreeHead: *sth, + StoredAt: time.Now(), // not the exact modtime of the file, but close enough for our purposes + }, nil +} + +func removeSTHFromDir(dirPath string, sth *cttypes.SignedTreeHead) error { + filePath := filepath.Join(dirPath, sthFilename(sth)) + err := os.Remove(filePath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +// generate a filename that uniquely identifies the STH (within the context of a particular log) +func sthFilename(sth *cttypes.SignedTreeHead) string { + hasher := sha256.New() + binary.Write(hasher, binary.LittleEndian, sth.Timestamp) + hasher.Write(sth.RootHash[:]) + return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json" +} + +// sthFilenameTreeSize returns the tree size encoded in the filename, or (0, false) +// if the filename doesn't parse. +func sthFilenameTreeSize(filename string) (uint64, bool) { + separator := strings.IndexByte(filename, '-') + if separator <= 0 { + return 0, false + } + treeSize, err := strconv.ParseUint(filename[:separator], 10, 64) + if err != nil { + return 0, false + } + return treeSize, true +} diff --git a/third_party/certspotter/monitor/watchlist.go b/third_party/certspotter/monitor/watchlist.go new file mode 100644 index 0000000..8a80852 --- /dev/null +++ b/third_party/certspotter/monitor/watchlist.go @@ -0,0 +1,135 @@ +// Copyright (C) 2016, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bufio" + "fmt" + "golang.org/x/net/idna" + "io" + "software.sslmate.com/src/certspotter" + "strings" +) + +type WatchItem struct { + domain []string + acceptSuffix bool +} + +type WatchList []WatchItem + +func ParseWatchItem(str string) (WatchItem, error) { + fields := strings.Fields(str) + if len(fields) == 0 { + return WatchItem{}, fmt.Errorf("empty domain") + } + domain := fields[0] + + for _, field := range fields[1:] { + switch { + case strings.HasPrefix(field, "valid_at:"): + // Ignore for backwards compatibility + default: + return WatchItem{}, fmt.Errorf("unknown parameter %q", field) + } + } + + if domain == "." { + // "." as in root zone -> matches everything + return WatchItem{ + domain: []string{}, + acceptSuffix: true, + }, nil + } + + acceptSuffix := false + if strings.HasPrefix(domain, ".") { + acceptSuffix = true + domain = domain[1:] + } + + asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, "."))) + if err != nil { + return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err) + } + return WatchItem{ + domain: strings.Split(asciiDomain, "."), + acceptSuffix: acceptSuffix, + }, nil +} + +func ReadWatchList(reader io.Reader) (WatchList, error) { + items := make(WatchList, 0, 50) + scanner := bufio.NewScanner(reader) + lineNo := 0 + for scanner.Scan() { + line := scanner.Text() + lineNo++ + if line == "" || strings.HasPrefix(line, "#") { + continue + } + item, err := ParseWatchItem(line) + if err != nil { + return nil, fmt.Errorf("%w on line %d", err, lineNo) + } + items = append(items, item) + } + return items, scanner.Err() +} + +func (item WatchItem) String() string { + if item.acceptSuffix { + return "." + strings.Join(item.domain, ".") + } else { + return strings.Join(item.domain, ".") + } +} + +func (item WatchItem) matchesDNSName(dnsName []string) bool { + watchDomain := item.domain + for len(dnsName) > 0 && len(watchDomain) > 0 { + certLabel := dnsName[len(dnsName)-1] + watchLabel := watchDomain[len(watchDomain)-1] + + if !dnsLabelMatches(certLabel, watchLabel) { + return false + } + + dnsName = dnsName[:len(dnsName)-1] + watchDomain = watchDomain[:len(watchDomain)-1] + } + return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0) +} + +func dnsLabelMatches(certLabel string, watchLabel string) bool { + // For fail-safe behavior, if a label was unparsable, it matches everything. + // Similarly, redacted labels match everything, since the label _might_ be + // for a name we're interested in. + + return certLabel == "*" || + certLabel == "?" || + certLabel == certspotter.UnparsableDNSLabelPlaceholder || + certspotter.MatchesWildcard(watchLabel, certLabel) +} + +func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) { + dnsNames := make([][]string, len(identifiers.DNSNames)) + for i, dnsName := range identifiers.DNSNames { + dnsNames[i] = strings.Split(dnsName, ".") + } + for _, item := range list { + for _, dnsName := range dnsNames { + if item.matchesDNSName(dnsName) { + return true, item + } + } + } + return false, WatchItem{} +} diff --git a/third_party/certspotter/precerts.go b/third_party/certspotter/precerts.go new file mode 100644 index 0000000..29755b4 --- /dev/null +++ b/third_party/certspotter/precerts.go @@ -0,0 +1,154 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "bytes" + "encoding/asn1" + "errors" + "fmt" +) + +func bitStringEqual(a, b *asn1.BitString) bool { + return a.BitLength == b.BitLength && bytes.Equal(a.Bytes, b.Bytes) +} + +var ( + oidExtensionAuthorityKeyId = []int{2, 5, 29, 35} + oidExtensionSCT = []int{1, 3, 6, 1, 4, 1, 11129, 2, 4, 2} + oidExtensionCTPoison = []int{1, 3, 6, 1, 4, 1, 11129, 2, 4, 3} +) + +type PrecertInfo struct { + SameIssuer bool // The pre-certificate was issued from the same CA as the final certificate + Issuer []byte // The pre-certificate's issuer, if different from the final certificate + AKI []byte // The pre-certificate's AKI, if present and different from the final certificate +} + +func ValidatePrecert(precertBytes []byte, tbsBytes []byte) (*PrecertInfo, error) { + precert, err := ParseCertificate(precertBytes) + if err != nil { + return nil, errors.New("failed to parse pre-certificate: " + err.Error()) + } + precertTBS, err := precert.ParseTBSCertificate() + if err != nil { + return nil, errors.New("failed to parse pre-certificate TBS: " + err.Error()) + } + tbs, err := ParseTBSCertificate(tbsBytes) + if err != nil { + return nil, errors.New("failed to parse TBS: " + err.Error()) + } + + // Everything must be equal except: + // issuer + // Authority Key Identifier extension (both must have it OR neither can have it) + // CT poison extension (precert must have it, TBS must not have it) + if precertTBS.Version != tbs.Version { + return nil, errors.New("version not equal") + } + if !bytes.Equal(precertTBS.SerialNumber.FullBytes, tbs.SerialNumber.FullBytes) { + return nil, errors.New("serial number not equal") + } + sameIssuer := bytes.Equal(precertTBS.Issuer.FullBytes, tbs.Issuer.FullBytes) + if !bytes.Equal(precertTBS.SignatureAlgorithm.FullBytes, tbs.SignatureAlgorithm.FullBytes) { + return nil, errors.New("SignatureAlgorithm not equal") + } + if !bytes.Equal(precertTBS.Validity.FullBytes, tbs.Validity.FullBytes) { + return nil, errors.New("Validity not equal") + } + if !bytes.Equal(precertTBS.Subject.FullBytes, tbs.Subject.FullBytes) { + return nil, errors.New("Subject not equal") + } + if !bytes.Equal(precertTBS.PublicKey.FullBytes, tbs.PublicKey.FullBytes) { + return nil, errors.New("PublicKey not equal") + } + if !bitStringEqual(&precertTBS.UniqueId, &tbs.UniqueId) { + return nil, errors.New("UniqueId not equal") + } + if !bitStringEqual(&precertTBS.SubjectUniqueId, &tbs.SubjectUniqueId) { + return nil, errors.New("SubjectUniqueId not equal") + } + + precertHasPoison := false + tbsIndex := 0 + var aki []byte + for precertIndex := range precertTBS.Extensions { + precertExt := &precertTBS.Extensions[precertIndex] + + if precertExt.Id.Equal(oidExtensionCTPoison) { + if !precertExt.Critical { + return nil, errors.New("pre-cert poison extension is not critical") + } + /* CAs can't even get this right, and Google's logs don't check. Fortunately, + it's not that important. + if !bytes.Equal(precertExt.Value, []byte{0x05, 0x00}) { + return errors.New("pre-cert poison extension contains incorrect value") + } + */ + precertHasPoison = true + continue + } + + if tbsIndex >= len(tbs.Extensions) { + return nil, errors.New("pre-cert contains extension not in TBS") + } + tbsExt := &tbs.Extensions[tbsIndex] + + if !precertExt.Id.Equal(tbsExt.Id) { + return nil, fmt.Errorf("pre-cert and TBS contain different extensions (%v vs %v)", precertExt.Id, tbsExt.Id) + } + if precertExt.Critical != tbsExt.Critical { + return nil, fmt.Errorf("pre-cert and TBS %v extension differs in criticality", precertExt.Id) + } + if !sameIssuer && precertExt.Id.Equal(oidExtensionAuthorityKeyId) { + aki = precertExt.Value + } else { + if !bytes.Equal(precertExt.Value, tbsExt.Value) { + return nil, fmt.Errorf("pre-cert and TBS %v extension differs in value", precertExt.Id) + } + } + + tbsIndex++ + } + if tbsIndex < len(tbs.Extensions) { + return nil, errors.New("TBS contains extension not in pre-cert") + } + if !precertHasPoison { + return nil, errors.New("pre-cert does not have poison extension") + } + + return &PrecertInfo{SameIssuer: sameIssuer, Issuer: precertTBS.Issuer.FullBytes, AKI: aki}, nil +} +func ReconstructPrecertTBS(tbs *TBSCertificate) (*TBSCertificate, error) { + precertTBS := TBSCertificate{ + Version: tbs.Version, + SerialNumber: tbs.SerialNumber, + SignatureAlgorithm: tbs.SignatureAlgorithm, + Issuer: tbs.Issuer, + Validity: tbs.Validity, + Subject: tbs.Subject, + PublicKey: tbs.PublicKey, + UniqueId: tbs.UniqueId, + SubjectUniqueId: tbs.SubjectUniqueId, + Extensions: make([]Extension, 0, len(tbs.Extensions)), + } + + for _, ext := range tbs.Extensions { + switch { + case ext.Id.Equal(oidExtensionSCT): + default: + precertTBS.Extensions = append(precertTBS.Extensions, ext) + } + } + + var err error + precertTBS.Raw, err = asn1.Marshal(precertTBS) + return &precertTBS, err +} diff --git a/third_party/certspotter/sequencer/sequencer.go b/third_party/certspotter/sequencer/sequencer.go new file mode 100644 index 0000000..1a53ad8 --- /dev/null +++ b/third_party/certspotter/sequencer/sequencer.go @@ -0,0 +1,151 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package sequencer + +import ( + "context" + "slices" + "sync" +) + +type seqWriter struct { + seqNbr uint64 + ready chan<- struct{} +} + +// Channel[T] is a multi-producer, single-consumer channel of items with monotonicaly-increasing sequence numbers. +// Items can be sent in any order, but they are always received in order of their sequence number. +// It is unsafe to call Next concurrently with itself, or to call Add/Reserve concurrently with another Add/Reserve +// call for the same sequence number. Otherwise, methods are safe to call concurrently. +type Channel[T any] struct { + mu sync.Mutex + next uint64 + buf []*T + writers []seqWriter + readWaiting bool + readReady chan struct{} +} + +func New[T any](initialSequenceNumber uint64, capacity uint64) *Channel[T] { + return &Channel[T]{ + buf: make([]*T, capacity), + next: initialSequenceNumber, + readReady: make(chan struct{}, 1), + } +} + +func (seq *Channel[T]) parkWriter(seqNbr uint64) <-chan struct{} { + ready := make(chan struct{}) + seq.writers = append(seq.writers, seqWriter{seqNbr: seqNbr, ready: ready}) + return ready +} + +func (seq *Channel[T]) signalWriter(seqNbr uint64) { + for i := range seq.writers { + if seq.writers[i].seqNbr == seqNbr { + close(seq.writers[i].ready) + seq.writers = slices.Delete(seq.writers, i, i+1) + return + } + } +} + +func (seq *Channel[T]) forgetWriter(seqNbr uint64) { + for i := range seq.writers { + if seq.writers[i].seqNbr == seqNbr { + seq.writers = slices.Delete(seq.writers, i, i+1) + return + } + } +} + +func (seq *Channel[T]) Cap() uint64 { + return uint64(len(seq.buf)) +} + +func (seq *Channel[T]) index(seqNbr uint64) int { + return int(seqNbr % seq.Cap()) +} + +// Wait until the channel has capacity for an item with the given sequence number. +// After this function returns nil, calling Add with the same sequence number will not block. +func (seq *Channel[T]) Reserve(ctx context.Context, sequenceNumber uint64) error { + seq.mu.Lock() + if sequenceNumber >= seq.next+seq.Cap() { + ready := seq.parkWriter(sequenceNumber) + seq.mu.Unlock() + select { + case <-ctx.Done(): + seq.mu.Lock() + seq.forgetWriter(sequenceNumber) + seq.mu.Unlock() + return ctx.Err() + case <-ready: + } + } else { + seq.mu.Unlock() + } + return nil +} + +// Send an item with the given sequence number. Blocks if the channel does not have capacity for the item. +// It is undefined behavior to send a sequence number that has previously been sent. +func (seq *Channel[T]) Add(ctx context.Context, sequenceNumber uint64, item *T) error { + seq.mu.Lock() + if sequenceNumber >= seq.next+seq.Cap() { + ready := seq.parkWriter(sequenceNumber) + seq.mu.Unlock() + select { + case <-ctx.Done(): + seq.mu.Lock() + seq.forgetWriter(sequenceNumber) + seq.mu.Unlock() + return ctx.Err() + case <-ready: + } + seq.mu.Lock() + } + seq.buf[seq.index(sequenceNumber)] = item + if sequenceNumber == seq.next && seq.readWaiting { + seq.readReady <- struct{}{} + } + seq.mu.Unlock() + return nil +} + +// Return the item with the next sequence number, blocking if necessary. +// Not safe to call concurrently with other Next calls. +func (seq *Channel[T]) Next(ctx context.Context) (*T, error) { + seq.mu.Lock() + if seq.buf[seq.index(seq.next)] == nil { + seq.readWaiting = true + seq.mu.Unlock() + select { + case <-ctx.Done(): + seq.mu.Lock() + select { + case <-seq.readReady: + default: + } + seq.readWaiting = false + seq.mu.Unlock() + return nil, ctx.Err() + case <-seq.readReady: + } + seq.mu.Lock() + seq.readWaiting = false + } + item := seq.buf[seq.index(seq.next)] + seq.buf[seq.index(seq.next)] = nil + seq.signalWriter(seq.next + seq.Cap()) + seq.next++ + seq.mu.Unlock() + return item, nil +} diff --git a/third_party/certspotter/sequencer/sequencer_test.go b/third_party/certspotter/sequencer/sequencer_test.go new file mode 100644 index 0000000..27a9211 --- /dev/null +++ b/third_party/certspotter/sequencer/sequencer_test.go @@ -0,0 +1,195 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package sequencer + +import ( + "context" + "fmt" + mathrand "math/rand/v2" + "testing" + "time" +) + +func TestSequencerBasic(t *testing.T) { + ctx := context.Background() + seq := New[uint64](0, 100) + go func() { + for i := range uint64(10_000) { + err := seq.Add(ctx, i, &i) + if err != nil { + panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err)) + } + } + }() + + for i := range uint64(10_000) { + next, err := seq.Next(ctx) + if err != nil { + t.Fatalf("%d: seq.Next returned unexpected error %v", i, err) + } + if *next != i { + t.Fatalf("%d: got unexpected value %d", i, *next) + } + } +} + +func TestSequencerNonZeroStart(t *testing.T) { + ctx := context.Background() + seq := New[uint64](10, 100) + go func() { + for i := range uint64(10_000) { + err := seq.Add(ctx, i+10, &i) + if err != nil { + panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err)) + } + } + }() + + for i := range uint64(10_000) { + next, err := seq.Next(ctx) + if err != nil { + t.Fatalf("%d: seq.Next returned unexpected error %v", i, err) + } + if *next != i { + t.Fatalf("%d: got unexpected value %d", i, *next) + } + } +} + +func TestSequencerCapacity1(t *testing.T) { + ctx := context.Background() + seq := New[uint64](0, 1) + go func() { + for i := range uint64(10_000) { + err := seq.Add(ctx, i, &i) + if err != nil { + panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err)) + } + } + }() + + for i := range uint64(10_000) { + next, err := seq.Next(ctx) + if err != nil { + t.Fatalf("%d: seq.Next returned unexpected error %v", i, err) + } + if *next != i { + t.Fatalf("%d: got unexpected value %d", i, *next) + } + } +} + +func TestSequencerTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + seq := New[uint64](0, 10_000) + go func() { + var i uint64 + for { + newI := i + err := seq.Add(ctx, i, &newI) + if err != nil { + break + } + i++ + } + }() + + var i uint64 + for { + next, err := seq.Next(ctx) + if err != nil { + break + } + if *next != i { + t.Fatalf("%d: got unexpected value %d", i, *next) + } + i++ + } +} + +func TestSequencerOutOfOrder(t *testing.T) { + ctx := context.Background() + seq := New[uint64](0, 100) + ch := make(chan uint64) + go func() { + for i := range uint64(10_000) { + ch <- i + } + }() + for range 4 { + go func() { + for i := range ch { + time.Sleep(mathrand.N(10 * time.Millisecond)) + //t.Logf("seq.Add %d", i) + err := seq.Add(ctx, i, &i) + if err != nil { + panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err)) + } + } + }() + } + for i := range uint64(10_000) { + next, err := seq.Next(ctx) + if err != nil { + t.Fatalf("%d: seq.Next returned unexpected error %v", i, err) + } + if *next != i { + t.Fatalf("%d: got unexpected value %d", i, *next) + } + //t.Logf("seq.Next %d", i) + } +} + +func TestSequencerOutOfOrderReserve(t *testing.T) { + ctx := context.Background() + seq := New[uint64](0, 10) + ch := make(chan uint64) + go func() { + for i := range uint64(10_000) { + ch <- i + } + }() + ch2 := make(chan uint64) + for job := range 4 { + go func() { + for i := range ch { + time.Sleep(mathrand.N(11 * time.Duration(job+1) * time.Millisecond)) + err := seq.Reserve(ctx, i) + if err != nil { + panic(fmt.Sprintf("%d: seq.Reserve returned unexpected error %v", i, err)) + } + ch2 <- i + } + }() + } + for range 4 { + go func() { + for i := range ch2 { + time.Sleep(mathrand.N(7 * time.Millisecond)) + t.Logf("seq.Add %d", i) + err := seq.Add(ctx, i, &i) + if err != nil { + panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err)) + } + } + }() + } + for i := range uint64(10_000) { + next, err := seq.Next(ctx) + if err != nil { + t.Fatalf("%d: seq.Next returned unexpected error %v", i, err) + } + if *next != i { + t.Fatalf("%d: got unexpected value %d", i, *next) + } + t.Logf("seq.Next %d", i) + } +} diff --git a/third_party/certspotter/staticcheck.conf b/third_party/certspotter/staticcheck.conf new file mode 100644 index 0000000..64f693e --- /dev/null +++ b/third_party/certspotter/staticcheck.conf @@ -0,0 +1 @@ +checks = ["inherit", "-ST1005", "-S1002"] diff --git a/third_party/certspotter/tlstypes/signatures.go b/third_party/certspotter/tlstypes/signatures.go new file mode 100644 index 0000000..90edb1a --- /dev/null +++ b/third_party/certspotter/tlstypes/signatures.go @@ -0,0 +1,134 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package tlstypes + +import ( + "bytes" + "encoding/json" + "fmt" + "golang.org/x/crypto/cryptobyte" +) + +type HashAlgorithm uint8 + +const ( + SHA224 HashAlgorithm = 3 + SHA256 HashAlgorithm = 4 + SHA384 HashAlgorithm = 5 + SHA512 HashAlgorithm = 6 +) + +type SignatureAlgorithm uint8 + +const ( + RSA SignatureAlgorithm = 1 + ECDSA SignatureAlgorithm = 3 +) + +type SignatureAndHashAlgorithm struct { + Hash HashAlgorithm + Signature SignatureAlgorithm +} + +type DigitallySigned struct { + Algorithm SignatureAndHashAlgorithm + Signature []byte +} + +func (v HashAlgorithm) Marshal(b *cryptobyte.Builder) error { + b.AddUint8(uint8(v)) + return nil +} +func (v *HashAlgorithm) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint8((*uint8)(v)) +} + +func (v SignatureAlgorithm) Marshal(b *cryptobyte.Builder) error { + b.AddUint8(uint8(v)) + return nil +} +func (v *SignatureAlgorithm) Unmarshal(s *cryptobyte.String) bool { + return s.ReadUint8((*uint8)(v)) +} + +func (v SignatureAndHashAlgorithm) Marshal(b *cryptobyte.Builder) error { + b.AddValue(v.Hash) + b.AddValue(v.Signature) + return nil +} +func (v *SignatureAndHashAlgorithm) Unmarshal(s *cryptobyte.String) bool { + return v.Hash.Unmarshal(s) && v.Signature.Unmarshal(s) +} + +func (v DigitallySigned) Marshal(b *cryptobyte.Builder) error { + b.AddValue(v.Algorithm) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { b.AddBytes(v.Signature) }) + return nil +} +func (v *DigitallySigned) Unmarshal(s *cryptobyte.String) bool { + return v.Algorithm.Unmarshal(s) && s.ReadUint16LengthPrefixed((*cryptobyte.String)(&v.Signature)) +} + +func (v DigitallySigned) Bytes() []byte { + b := cryptobyte.NewBuilder(make([]byte, 0, 4+len(v.Signature))) + b.AddValue(v) + return b.BytesOrPanic() +} + +func (v DigitallySigned) MarshalBinary() ([]byte, error) { + b := cryptobyte.NewBuilder(make([]byte, 0, 4+len(v.Signature))) + b.AddValue(v) + return b.Bytes() +} + +func (v *DigitallySigned) UnmarshalBinary(data []byte) error { + str := cryptobyte.String(bytes.Clone(data)) + if !v.Unmarshal(&str) { + return fmt.Errorf("DigitallySigned bytes are malformed") + } + if !str.Empty() { + return fmt.Errorf("trailing bytes after DigitallySigned") + } + return nil +} + +func (v DigitallySigned) MarshalJSON() ([]byte, error) { + b := cryptobyte.NewBuilder(make([]byte, 0, 4+len(v.Signature))) + b.AddValue(v) + if bytes, err := b.Bytes(); err != nil { + return nil, err + } else { + return json.Marshal(bytes) + } +} +func (v *DigitallySigned) UnmarshalJSON(data []byte) error { + str := new(cryptobyte.String) + if err := json.Unmarshal(data, (*[]byte)(str)); err != nil { + return fmt.Errorf("unable to unmarshal DigitallySigned JSON: %w", err) + } + if !v.Unmarshal(str) { + return fmt.Errorf("DigitallySigned bytes are malformed") + } + if !str.Empty() { + return fmt.Errorf("trailing bytes after DigitallySigned") + } + return nil +} +func ParseDigitallySigned(bytes []byte) (*DigitallySigned, error) { + ds := new(DigitallySigned) + str := cryptobyte.String(bytes) + if !ds.Unmarshal(&str) { + return nil, fmt.Errorf("DigitallySigned bytes are malformed") + } + if !str.Empty() { + return nil, fmt.Errorf("trailing bytes after DigitallySigned") + } + return ds, nil +} diff --git a/third_party/certspotter/x509.go b/third_party/certspotter/x509.go new file mode 100644 index 0000000..b2b6fdf --- /dev/null +++ b/third_party/certspotter/x509.go @@ -0,0 +1,441 @@ +// Copyright (C) 2016 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "bytes" + "crypto/x509/pkix" + "encoding/asn1" + "errors" + "fmt" + "math/big" + "net" + "time" + "unicode" +) + +var ( + oidExtensionSubjectAltName = asn1.ObjectIdentifier{2, 5, 29, 17} + oidExtensionBasicConstraints = asn1.ObjectIdentifier{2, 5, 29, 19} + oidCountry = asn1.ObjectIdentifier{2, 5, 4, 6} + oidOrganization = asn1.ObjectIdentifier{2, 5, 4, 10} + oidOrganizationalUnit = asn1.ObjectIdentifier{2, 5, 4, 11} + oidCommonName = asn1.ObjectIdentifier{2, 5, 4, 3} + oidSerialNumber = asn1.ObjectIdentifier{2, 5, 4, 5} + oidLocality = asn1.ObjectIdentifier{2, 5, 4, 7} + oidProvince = asn1.ObjectIdentifier{2, 5, 4, 8} + oidStreetAddress = asn1.ObjectIdentifier{2, 5, 4, 9} + oidPostalCode = asn1.ObjectIdentifier{2, 5, 4, 17} +) + +type CertValidity struct { + NotBefore time.Time + NotAfter time.Time +} + +type basicConstraints struct { + IsCA bool `asn1:"optional"` + MaxPathLen int `asn1:"optional,default:-1"` +} + +type Extension struct { + Id asn1.ObjectIdentifier + Critical bool `asn1:"optional"` + Value []byte +} + +const ( + sanOtherName = 0 + sanRfc822Name = 1 + sanDNSName = 2 + sanX400Address = 3 + sanDirectoryName = 4 + sanEdiPartyName = 5 + sanURI = 6 + sanIPAddress = 7 + sanRegisteredID = 8 +) + +type SubjectAltName struct { + Type int + Value []byte +} + +type RDNSequence []RelativeDistinguishedNameSET +type RelativeDistinguishedNameSET []AttributeTypeAndValue +type AttributeTypeAndValue struct { + Type asn1.ObjectIdentifier + Value asn1.RawValue +} + +func ParseRDNSequence(rdnsBytes []byte) (RDNSequence, error) { + var rdns RDNSequence + if rest, err := asn1.Unmarshal(rdnsBytes, &rdns); err != nil { + return nil, errors.New("failed to parse RDNSequence: " + err.Error()) + } else if len(rest) != 0 { + return nil, fmt.Errorf("trailing data after RDNSequence: %v", rest) // XXX: too strict? + } + return rdns, nil +} + +func MarshalRDNSequence(rdns RDNSequence) ([]byte, error) { + return asn1.Marshal(rdns) +} + +type TBSCertificate struct { + Raw asn1.RawContent + + Version int `asn1:"optional,explicit,default:1,tag:0"` + SerialNumber asn1.RawValue + SignatureAlgorithm asn1.RawValue + Issuer asn1.RawValue + Validity asn1.RawValue + Subject asn1.RawValue + PublicKey asn1.RawValue + UniqueId asn1.BitString `asn1:"optional,tag:1"` + SubjectUniqueId asn1.BitString `asn1:"optional,tag:2"` + Extensions []Extension `asn1:"optional,explicit,tag:3"` +} + +type Certificate struct { + Raw asn1.RawContent + + TBSCertificate asn1.RawValue + SignatureAlgorithm asn1.RawValue + SignatureValue asn1.RawValue +} + +func (rdns RDNSequence) ParseCNs() ([]string, error) { + var cns []string + + // An RDN is a SET OF AttributeTypeAndValue, so the commonName attribute + // is not necessarily the first (or only) element. Examine every + // attribute so that a CN doesn't escape detection by being placed in a + // non-first position of a multi-valued RDN. + for _, rdn := range rdns { + for _, atv := range rdn { + if atv.Type.Equal(oidCommonName) { + cnString, err := decodeASN1String(&atv.Value) + if err != nil { + return nil, errors.New("Error decoding CN: " + err.Error()) + } + cns = append(cns, cnString) + } + } + } + + return cns, nil +} + +func rdnLabel(oid asn1.ObjectIdentifier) string { + switch { + case oid.Equal(oidCountry): + return "C" + case oid.Equal(oidOrganization): + return "O" + case oid.Equal(oidOrganizationalUnit): + return "OU" + case oid.Equal(oidCommonName): + return "CN" + case oid.Equal(oidSerialNumber): + return "serialNumber" + case oid.Equal(oidLocality): + return "L" + case oid.Equal(oidProvince): + return "ST" + case oid.Equal(oidStreetAddress): + return "street" + case oid.Equal(oidPostalCode): + return "postalCode" + } + return oid.String() +} + +func (rdns RDNSequence) String() string { + var buf bytes.Buffer + + for _, rdn := range rdns { + // An RDN is a SET OF AttributeTypeAndValue; render every attribute, + // joining the attributes of a multi-valued RDN with '+' and separate + // RDNs with ", " (per RFC 4514). + for j, atv := range rdn { + if buf.Len() != 0 { + if j == 0 { + buf.WriteString(", ") + } else { + buf.WriteByte('+') + } + } + buf.WriteString(rdnLabel(atv.Type)) + buf.WriteString("=") + valueString, err := decodeASN1String(&atv.Value) + if err == nil { + buf.WriteString(escapeDNString(valueString)) + } else { + fmt.Fprintf(&buf, "%v", atv.Value.FullBytes) + } + } + } + + return buf.String() +} + +// escapeDNString escapes a distinguished name attribute value for safe display +// and for safe use as a hook script environment variable value. It escapes the +// backslash and comma characters (so the value is unambiguous within a +// comma-separated RDNSequence) and replaces non-printable characters with a +// backslash hex escape. In particular, this ensures the value never contains a +// NUL byte, which would otherwise be rejected by os/exec when passed as an +// environment variable and abort the notification. +func escapeDNString(value string) string { + var buf bytes.Buffer + for _, r := range value { + switch { + case r == '\\' || r == ',': + buf.WriteByte('\\') + buf.WriteRune(r) + case unicode.IsPrint(r): + buf.WriteRune(r) + case r <= 0xFF: + fmt.Fprintf(&buf, "\\x%02x", r) + case r <= 0xFFFF: + fmt.Fprintf(&buf, "\\u%04x", r) + default: + fmt.Fprintf(&buf, "\\U%08x", r) + } + } + return buf.String() +} + +func (san SubjectAltName) String() string { + switch san.Type { + case sanDNSName: + return "DNS:" + escapeDNString(string(san.Value)) + case sanIPAddress: + if len(san.Value) == 4 || len(san.Value) == 16 { + return "IP:" + net.IP(san.Value).String() + } else { + return fmt.Sprintf("IP:%v", san.Value) + } + default: + // TODO: support other types of SANs + return fmt.Sprintf("%d:%v", san.Type, san.Value) + } +} + +func ParseTBSCertificate(tbsBytes []byte) (*TBSCertificate, error) { + var tbs TBSCertificate + if rest, err := asn1.Unmarshal(tbsBytes, &tbs); err != nil { + return nil, errors.New("failed to parse TBS: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after TBS: %v", rest) // XXX: too strict? + } + return &tbs, nil +} + +func (tbs *TBSCertificate) ParseValidity() (*CertValidity, error) { + var rawValidity struct { + NotBefore asn1.RawValue + NotAfter asn1.RawValue + } + if rest, err := asn1.Unmarshal(tbs.Validity.FullBytes, &rawValidity); err != nil { + return nil, errors.New("failed to parse validity: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after validity: %v", rest) + } + + var validity CertValidity + var err error + if validity.NotBefore, err = decodeASN1Time(&rawValidity.NotBefore); err != nil { + return nil, errors.New("failed to decode notBefore time: " + err.Error()) + } + if validity.NotAfter, err = decodeASN1Time(&rawValidity.NotAfter); err != nil { + return nil, errors.New("failed to decode notAfter time: " + err.Error()) + } + + return &validity, nil +} + +func (tbs *TBSCertificate) ParseBasicConstraints() (*bool, error) { + isCA := false + isNotCA := false + + // Some certs in the wild have multiple BasicConstraints extensions (is there anything + // that CAs haven't screwed up???), so we process all of them and only choke if they + // are contradictory (which has not been observed...yet). + for _, ext := range tbs.GetExtension(oidExtensionBasicConstraints) { + var constraints basicConstraints + if rest, err := asn1.Unmarshal(ext.Value, &constraints); err != nil { + return nil, errors.New("failed to parse Basic Constraints: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after Basic Constraints: %v", rest) + } + + if constraints.IsCA { + isCA = true + } else { + isNotCA = true + } + } + + if !isCA && !isNotCA { + return nil, nil + } else if isCA && !isNotCA { + trueValue := true + return &trueValue, nil + } else if !isCA && isNotCA { + falseValue := false + return &falseValue, nil + } else { + return nil, fmt.Errorf("Certificate has more than one Basic Constraints extension and they are contradictory") + } +} + +func (tbs *TBSCertificate) ParseSerialNumber() (*big.Int, error) { + serialNumber := big.NewInt(0) + if rest, err := asn1.Unmarshal(tbs.SerialNumber.FullBytes, &serialNumber); err != nil { + return nil, errors.New("failed to parse serial number: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after serial number: %v", rest) + } + return serialNumber, nil +} + +func (tbs *TBSCertificate) GetRawPublicKey() []byte { + return tbs.PublicKey.FullBytes +} + +func (tbs *TBSCertificate) GetRawSubject() []byte { + return tbs.Subject.FullBytes +} + +func (tbs *TBSCertificate) GetRawIssuer() []byte { + return tbs.Issuer.FullBytes +} + +func (tbs *TBSCertificate) ParseSubject() (RDNSequence, error) { + subject, err := ParseRDNSequence(tbs.GetRawSubject()) + if err != nil { + return nil, errors.New("failed to parse certificate subject: " + err.Error()) + } + return subject, nil +} + +func (tbs *TBSCertificate) ParseIssuer() (RDNSequence, error) { + issuer, err := ParseRDNSequence(tbs.GetRawIssuer()) + if err != nil { + return nil, errors.New("failed to parse certificate issuer: " + err.Error()) + } + return issuer, nil +} + +func (tbs *TBSCertificate) ParseSubjectCommonNames() ([]string, error) { + subject, err := tbs.ParseSubject() + if err != nil { + return nil, err + } + cns, err := subject.ParseCNs() + if err != nil { + return nil, errors.New("failed to process certificate subject: " + err.Error()) + } + + return cns, nil +} + +func (tbs *TBSCertificate) ParseSubjectAltNames() ([]SubjectAltName, error) { + sans := []SubjectAltName{} + + for _, sanExt := range tbs.GetExtension(oidExtensionSubjectAltName) { + var err error + sans, err = ParseSANExtension(sans, sanExt.Value) + if err != nil { + return nil, err + } + } + + return sans, nil +} + +func (tbs *TBSCertificate) GetExtension(id asn1.ObjectIdentifier) []Extension { + var exts []Extension + for _, ext := range tbs.Extensions { + if ext.Id.Equal(id) { + exts = append(exts, ext) + } + } + return exts +} + +func ParseCertificate(certBytes []byte) (*Certificate, error) { + var cert Certificate + if rest, err := asn1.Unmarshal(certBytes, &cert); err != nil { + return nil, errors.New("failed to parse certificate: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after certificate: %v", rest) // XXX: too strict? + } + return &cert, nil +} + +func (cert *Certificate) GetRawTBSCertificate() []byte { + return cert.TBSCertificate.FullBytes +} + +func (cert *Certificate) ParseTBSCertificate() (*TBSCertificate, error) { + return ParseTBSCertificate(cert.GetRawTBSCertificate()) +} + +func (cert *Certificate) ParseSignatureAlgorithm() (*pkix.AlgorithmIdentifier, error) { + signatureAlgorithm := new(pkix.AlgorithmIdentifier) + if rest, err := asn1.Unmarshal(cert.SignatureAlgorithm.FullBytes, signatureAlgorithm); err != nil { + return nil, errors.New("failed to parse signature algorithm: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after signature algorithm: %v", rest) + } + return signatureAlgorithm, nil +} + +func (cert *Certificate) ParseSignatureValue() ([]byte, error) { + var signatureValue asn1.BitString + if rest, err := asn1.Unmarshal(cert.SignatureValue.FullBytes, &signatureValue); err != nil { + return nil, errors.New("failed to parse signature value: " + err.Error()) + } else if len(rest) > 0 { + return nil, fmt.Errorf("trailing data after signature value: %v", rest) + } + return signatureValue.RightAlign(), nil +} + +func ParseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) { + var seq asn1.RawValue + if rest, err := asn1.Unmarshal(value, &seq); err != nil { + return nil, errors.New("failed to parse subjectAltName extension: " + err.Error()) + } else if len(rest) != 0 { + // Don't complain if the SAN is followed by exactly one zero byte, + // which is a common error. + if !(len(rest) == 1 && rest[0] == 0) { + return nil, fmt.Errorf("trailing data in subjectAltName extension: %v", rest) // XXX: too strict? + } + } + if !seq.IsCompound || seq.Tag != 16 || seq.Class != 0 { + return nil, errors.New("failed to parse subjectAltName extension: bad SAN sequence") // XXX: too strict? + } + + rest := seq.Bytes + for len(rest) > 0 { + var val asn1.RawValue + var err error + rest, err = asn1.Unmarshal(rest, &val) + if err != nil { + return nil, errors.New("failed to parse subjectAltName extension item: " + err.Error()) + } + sans = append(sans, SubjectAltName{Type: val.Tag, Value: val.Bytes}) + } + + return sans, nil +} diff --git a/third_party/certspotter/x509_test.go b/third_party/certspotter/x509_test.go new file mode 100644 index 0000000..d0b3ac0 --- /dev/null +++ b/third_party/certspotter/x509_test.go @@ -0,0 +1,135 @@ +// Copyright (C) 2026 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package certspotter + +import ( + "encoding/asn1" + "os/exec" + "slices" + "strings" + "testing" +) + +// printableString builds an asn1.RawValue holding a PrintableString, as +// decodeASN1String expects to see it. +func printableString(s string) asn1.RawValue { + return asn1.RawValue{Class: 0, Tag: 19, IsCompound: false, Bytes: []byte(s)} +} + +func atv(oid asn1.ObjectIdentifier, value string) AttributeTypeAndValue { + return AttributeTypeAndValue{Type: oid, Value: printableString(value)} +} + +// A commonName that is not the first attribute of a multi-valued RDN must still +// be extracted, otherwise a watched name could escape detection. +func TestParseCNsMultiValuedRDN(t *testing.T) { + rdns := RDNSequence{ + { + atv(oidOrganization, "Example Org"), + atv(oidCommonName, "watched.example.com"), + }, + } + cns, err := rdns.ParseCNs() + if err != nil { + t.Fatal(err) + } + if want := []string{"watched.example.com"}; !slices.Equal(cns, want) { + t.Errorf("ParseCNs() = %q, want %q", cns, want) + } +} + +func TestParseCNsMultipleCNs(t *testing.T) { + rdns := RDNSequence{ + {atv(oidCommonName, "a.example.com")}, + { + atv(oidOrganizationalUnit, "Dept"), + atv(oidCommonName, "b.example.com"), + }, + } + cns, err := rdns.ParseCNs() + if err != nil { + t.Fatal(err) + } + if want := []string{"a.example.com", "b.example.com"}; !slices.Equal(cns, want) { + t.Errorf("ParseCNs() = %q, want %q", cns, want) + } +} + +func TestRDNSequenceStringEscaping(t *testing.T) { + tests := []struct { + name string + rdns RDNSequence + want string + }{ + { + name: "nul byte", + rdns: RDNSequence{{atv(oidCommonName, "ev\x00il")}}, + want: `CN=ev\x00il`, + }, + { + name: "backslash and comma", + rdns: RDNSequence{{atv(oidOrganization, `a\b,c`)}}, + want: `O=a\\b\,c`, + }, + { + name: "control characters", + rdns: RDNSequence{{atv(oidCommonName, "a\tb\nc")}}, + want: `CN=a\x09b\x0ac`, + }, + { + name: "multi-valued rdn joined with plus", + rdns: RDNSequence{{ + atv(oidOrganization, "Org"), + atv(oidCommonName, "host.example.com"), + }}, + want: "O=Org+CN=host.example.com", + }, + { + name: "separate rdns joined with comma", + rdns: RDNSequence{ + {atv(oidCommonName, "host.example.com")}, + {atv(oidOrganization, "Org")}, + }, + want: "CN=host.example.com, O=Org", + }, + { + name: "printable unicode passes through", + rdns: RDNSequence{{AttributeTypeAndValue{ + Type: oidOrganization, + Value: asn1.RawValue{Class: 0, Tag: 12, Bytes: []byte("café")}, // UTF8String + }}}, + want: "O=café", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.rdns.String(); got != tt.want { + t.Errorf("String() = %q, want %q", got, tt.want) + } + }) + } +} + +// Regression test for the denial of service in which a NUL byte in a certificate +// Subject/Issuer DN, placed into a hook script's environment, was rejected by +// os/exec and aborted notification (crashing the whole monitor). The escaped DN +// must be usable as an environment variable value. +func TestRDNSequenceStringIsExecSafe(t *testing.T) { + rdns := RDNSequence{{atv(oidCommonName, "ev\x00il\x01\x02")}} + dn := rdns.String() + if strings.ContainsRune(dn, 0) { + t.Fatalf("DN string still contains a NUL byte: %q", dn) + } + cmd := exec.Command("/bin/sh", "-c", "exit 0") + cmd.Env = []string{"SUBJECT_DN=" + dn} + if err := cmd.Run(); err != nil { + t.Fatalf("exec with SUBJECT_DN=%q failed: %v", dn, err) + } +}