ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

third_party/certspotter/ctclient/client.go

@@ -0,0 +1,204 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+// Package ctclient implements a client for monitoring RFC6962 and static-ct-api Certificate Transparency logs
+package ctclient
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+var UserAgent = ""
+
+// Create an HTTP client suitable for communicating with CT logs.  dialContext, if non-nil, is used for dialing.
+func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn, error)) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy:                 http.ProxyFromEnvironment,
+			TLSHandshakeTimeout:   15 * time.Second,
+			ResponseHeaderTimeout: 30 * time.Second,
+			MaxIdleConnsPerHost:   10,
+			IdleConnTimeout:       90 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSClientConfig: &tls.Config{
+				// We have to disable TLS certificate validation because because several logs
+				// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
+				// Since we verify that every response we receive from the log is signed
+				// by the log's CT public key (either directly, or indirectly via the Merkle Tree),
+				// TLS certificate validation is not actually necessary.  (We don't want to manage
+				// our own trust store because that adds undesired complexity and would require
+				// updating should a log ever change to a different CA.)
+				InsecureSkipVerify: true,
+			},
+			DialContext:       dialContext,
+			ForceAttemptHTTP2: true,
+		},
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return errors.New("redirects not followed")
+		},
+		Timeout: 60 * time.Second,
+	}
+}
+
+var defaultHTTPClient = NewHTTPClient(nil)
+
+// maxResponseBytes is a generous upper bound on the size of an HTTP response
+// body that we are willing to buffer in memory.
+const maxResponseBytes = 64 << 20 // 64 MiB
+
+// maxErrorResponseBytes is longest error response that we'll read
+const maxErrorResponseBytes = 8 << 10 // 8 KiB
+
+// readResponseBody reads a response body, returning an error if the
+// body is larger than maxBytes.
+func readResponseBody(r io.Reader, maxBytes int64) ([]byte, error) {
+	body, err := io.ReadAll(io.LimitReader(r, maxBytes+1))
+	if err != nil {
+		return nil, fmt.Errorf("error reading response: %w", err)
+	}
+	if int64(len(body)) > maxBytes {
+		return nil, fmt.Errorf("response body exceeds maximum allowed size of %d bytes", maxBytes)
+	}
+	return body, nil
+}
+
+func get(ctx context.Context, httpClient *http.Client, fullURL string) (io.ReadCloser, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+
+	if httpClient == nil {
+		httpClient = defaultHTTPClient
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	if response.StatusCode != 200 {
+		defer response.Body.Close()
+		responseBody, _ := io.ReadAll(io.LimitReader(response.Body, maxErrorResponseBytes))
+		return nil, fmt.Errorf("Get %q: %s (%s)", fullURL, response.Status, formatResponseBody(responseBody))
+	}
+
+	return response.Body, nil
+}
+
+func getBytes(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte, error) {
+	body, err := get(ctx, httpClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+	bodyBytes, err := readResponseBody(body, maxResponseBytes)
+	if err != nil {
+		return nil, fmt.Errorf("Get %q: %w", fullURL, err)
+	}
+	return bodyBytes, nil
+}
+
+func getJSON(ctx context.Context, httpClient *http.Client, fullURL string, response any) error {
+	responseBytes, err := getBytes(ctx, httpClient, fullURL)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(responseBytes, response); err != nil {
+		return fmt.Errorf("Get %q: error parsing response JSON: %w", fullURL, err)
+	}
+	return nil
+}
+
+func getRoots(ctx context.Context, httpClient *http.Client, logURL *url.URL) ([][]byte, error) {
+	fullURL := logURL.JoinPath("/ct/v1/get-roots").String()
+	var parsedResponse struct {
+		Certificates [][]byte `json:"certificates"`
+	}
+	if err := getJSON(ctx, httpClient, fullURL, &parsedResponse); err != nil {
+		return nil, err
+	}
+	return parsedResponse.Certificates, nil
+}
+
+func addChainOrPreChain(ctx context.Context, httpClient *http.Client, logURL *url.URL, isPreChain bool, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	var endpoint string
+	if isPreChain {
+		endpoint = "/ct/v1/add-pre-chain"
+	} else {
+		endpoint = "/ct/v1/add-chain"
+	}
+	fullURL := logURL.JoinPath(endpoint).String()
+
+	requestBody, err := json.Marshal(struct {
+		Chain [][]byte `json:"chain"`
+	}{
+		Chain: chain,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(requestBody))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+	request.Header.Set("Content-Type", "application/json")
+
+	if httpClient == nil {
+		httpClient = defaultHTTPClient
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	responseBody, err := readResponseBody(response.Body, maxResponseBytes)
+	if err != nil {
+		return nil, fmt.Errorf("Post %q: %w", fullURL, err)
+	}
+
+	if response.StatusCode != 200 {
+		return nil, fmt.Errorf("Post %q: %s (%s)", fullURL, response.Status, formatResponseBody(responseBody))
+	}
+
+	sct := new(cttypes.SignedCertificateTimestamp)
+	if err := json.Unmarshal(responseBody, sct); err != nil {
+		return nil, fmt.Errorf("Post %q: error parsing response JSON: %w", fullURL, err)
+	}
+
+	return sct, nil
+}
+
+func formatResponseBody(body []byte) string {
+	const maxLen = 200
+	body = bytes.TrimSpace(body)
+	if len(body) > maxLen {
+		return strconv.QuoteToASCII(string(body[:maxLen])) + "..."
+	} else {
+		return strconv.QuoteToASCII(string(body))
+	}
+}

third_party/certspotter/ctclient/client_test.go

@@ -0,0 +1,77 @@
+// Copyright (C) 2026 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestReadResponseBody(t *testing.T) {
+	// under the limit
+	if body, err := readResponseBody(strings.NewReader("hello"), 10); err != nil {
+		t.Errorf("unexpected error under limit: %v", err)
+	} else if string(body) != "hello" {
+		t.Errorf("got %q, want %q", body, "hello")
+	}
+
+	// exactly at the limit
+	if body, err := readResponseBody(strings.NewReader("0123456789"), 10); err != nil {
+		t.Errorf("unexpected error at exact limit: %v", err)
+	} else if len(body) != 10 {
+		t.Errorf("got %d bytes, want 10", len(body))
+	}
+
+	// one byte over the limit
+	if _, err := readResponseBody(strings.NewReader("0123456789X"), 10); err == nil {
+		t.Errorf("expected error for oversize body, got nil")
+	}
+}
+
+// Confirm the cap is actually wired into getBytes(): a body larger than the limit is
+// rejected with an error instead of being buffered in full.
+func TestGetEnforcesResponseLimit(t *testing.T) {
+	const bodyLen = maxResponseBytes + 1024
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		buf := make([]byte, 32*1024)
+		for written := 0; written < bodyLen; written += len(buf) {
+			if _, err := w.Write(buf); err != nil {
+				return
+			}
+		}
+	}))
+	defer server.Close()
+
+	if _, err := getBytes(context.Background(), server.Client(), server.URL); err == nil {
+		t.Errorf("getBytes() accepted an over-limit response body; expected an error")
+	} else if !strings.Contains(err.Error(), "maximum allowed size") {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+// A normal-sized response still round-trips correctly.
+func TestGetReadsNormalBody(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "tile-data")
+	}))
+	defer server.Close()
+
+	body, err := getBytes(context.Background(), server.Client(), server.URL)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if string(body) != "tile-data" {
+		t.Errorf("got %q, want %q", body, "tile-data")
+	}
+}

third_party/certspotter/ctclient/log.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type WritableLog interface {
+	AddChain(context.Context, [][]byte) (*cttypes.SignedCertificateTimestamp, error)
+	AddPreChain(context.Context, [][]byte) (*cttypes.SignedCertificateTimestamp, error)
+	GetRoots(context.Context) ([][]byte, error)
+}
+
+type Log interface {
+	GetSTH(context.Context) (*cttypes.SignedTreeHead, string, error)
+	GetRoots(context.Context) ([][]byte, error)
+	GetEntries(ctx context.Context, startInclusive, endInclusive uint64) ([]Entry, error)
+	ReconstructTree(context.Context, *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error)
+}
+
+// IssuerGetter represents a source of issuer certificates.
+//
+// If a [Log] also implements IssuerGetter, then it is mandatory to provide
+// an IssuerGetter when using [Entry]s returned by the [Log].  The IssuerGetter
+// may be the Log itself, or your own implementation which retrieves issuers
+// from a different source, such as a cache.
+//
+// If a Log doesn't implement IssuerGetter, then you may pass a nil IssuerGetter
+// when using the Log's Entrys.
+type IssuerGetter interface {
+	GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error)
+}
+
+type Entry interface {
+	LeafInput() []byte
+
+	// Returns error from IssuerGetter, otherwise infallible
+	ExtraData(context.Context, IssuerGetter) ([]byte, error)
+
+	// Returns an error if this is not a well-formed precert entry
+	Precertificate() (cttypes.ASN1Cert, error)
+
+	// Returns an error if this is not a well-formed x509 or precert entry
+	ChainFingerprints() ([][32]byte, error)
+
+	// Returns an error if this is not a well-formed x509 or precert entry, or if IssuerGetter failed
+	GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error)
+}

third_party/certspotter/ctclient/rfc6962.go

@@ -0,0 +1,272 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"slices"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type RFC6962Log struct {
+	URL        *url.URL
+	HTTPClient *http.Client // nil to use default client
+}
+
+type RFC6962LogEntry struct {
+	Leaf_input []byte `json:"leaf_input"`
+	Extra_data []byte `json:"extra_data"`
+}
+
+func (ctlog *RFC6962Log) AddChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.URL, false, chain)
+}
+
+func (ctlog *RFC6962Log) AddPreChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.URL, true, chain)
+}
+
+func (ctlog *RFC6962Log) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-sth").String()
+	sth := new(cttypes.SignedTreeHead)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, sth); err != nil {
+		return nil, fullURL, err
+	}
+	return sth, fullURL, nil
+}
+
+func (ctlog *RFC6962Log) GetRoots(ctx context.Context) ([][]byte, error) {
+	return getRoots(ctx, ctlog.HTTPClient, ctlog.URL)
+}
+
+func (ctlog *RFC6962Log) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]RFC6962LogEntry, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-entries").String()
+	fullURL += fmt.Sprintf("?start=%d&end=%d", startInclusive, endInclusive)
+
+	body, err := get(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+
+	limitedBody := io.LimitReader(body, maxResponseBytes)
+	dec := json.NewDecoder(limitedBody)
+	if tok, err := dec.Token(); err != nil {
+		return nil, fmt.Errorf("Get %q: error reading JSON object token: %w", fullURL, err)
+	} else if tok != json.Delim('{') {
+		return nil, fmt.Errorf("Get %q: not a JSON object", fullURL)
+	}
+	for {
+		if !dec.More() {
+			return nil, fmt.Errorf("Get %q: JSON object missing entries", fullURL)
+		}
+		tok, err := dec.Token()
+		if err != nil {
+			return nil, fmt.Errorf("Get %q: error reading JSON field name: %w", fullURL, err)
+		}
+		if tok == "entries" {
+			break
+		}
+		var rawMessage json.RawMessage
+		if err := dec.Decode(&rawMessage); err != nil {
+			return nil, fmt.Errorf("Get %q: error skipping unwanted JSON field: %w", fullURL, err)
+		}
+	}
+	if tok, err := dec.Token(); err != nil {
+		return nil, fmt.Errorf("Get %q: error reading JSON array token: %w", fullURL, err)
+	} else if tok != json.Delim('[') {
+		return nil, fmt.Errorf("Get %q: entries is not a JSON array", fullURL)
+	}
+
+	var entries []RFC6962LogEntry
+	for dec.More() {
+		if uint64(len(entries)) == endInclusive-startInclusive+1 {
+			return entries, fmt.Errorf("Get %q: extraneous entries returned", fullURL)
+		}
+		var entry RFC6962LogEntry
+		if err := dec.Decode(&entry); err != nil {
+			return entries, fmt.Errorf("Get %q: error reading JSON entry: %w", fullURL, err)
+		}
+		entries = append(entries, entry)
+	}
+	if tok, err := dec.Token(); err != nil {
+		return entries, fmt.Errorf("Get %q: error reading end of JSON array: %w", fullURL, err)
+	} else if tok != json.Delim(']') {
+		return entries, fmt.Errorf("Get %q: entries array not terminated", fullURL)
+	}
+	if len(entries) == 0 {
+		return entries, fmt.Errorf("Get %q: zero entries returned", fullURL)
+	}
+	io.Copy(io.Discard, limitedBody) // So the HTTP connection can be reused
+	return entries, nil
+}
+
+func (ctlog *RFC6962Log) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) {
+	nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive)
+	var entries []Entry
+	if nativeEntries != nil {
+		entries = make([]Entry, len(nativeEntries))
+		for i := range nativeEntries {
+			entries[i] = &nativeEntries[i]
+		}
+	}
+	return entries, err
+}
+
+type entryAndProofResponse struct {
+	LeafInput []byte            `json:"leaf_input"`
+	ExtraData []byte            `json:"extra_data"`
+	AuditPath []merkletree.Hash `json:"audit_path"`
+}
+
+func (ctlog *RFC6962Log) getEntryAndProof(ctx context.Context, leafIndex uint64, treeSize uint64) (*entryAndProofResponse, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-entry-and-proof").String()
+	fullURL += fmt.Sprintf("?leaf_index=%d&tree_size=%d", leafIndex, treeSize)
+	response := new(entryAndProofResponse)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+type proofResponse struct {
+	LeafIndex uint64            `json:"leaf_index"`
+	AuditPath []merkletree.Hash `json:"audit_path"`
+}
+
+func (ctlog *RFC6962Log) getProofByHash(ctx context.Context, hash *merkletree.Hash, treeSize uint64) (*proofResponse, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-proof-by-hash").String()
+	fullURL += fmt.Sprintf("?hash=%s&tree_size=%d", url.QueryEscape(hash.Base64String()), treeSize)
+	response := new(proofResponse)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (ctlog *RFC6962Log) reconstructTree(ctx context.Context, treeSize uint64) (*merkletree.CollapsedTree, error) {
+	if treeSize == 0 {
+		return new(merkletree.CollapsedTree), nil
+	}
+	if entryAndProof, err := ctlog.getEntryAndProof(ctx, treeSize-1, treeSize); err == nil {
+		tree := new(merkletree.CollapsedTree)
+		slices.Reverse(entryAndProof.AuditPath)
+		if err := tree.Init(entryAndProof.AuditPath, treeSize-1); err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for entry %d to STH %d: %w", treeSize-1, treeSize, err)
+		}
+		tree.Add(merkletree.HashLeaf(entryAndProof.LeafInput))
+		return tree, nil
+	}
+	entries, err := ctlog.getEntries(ctx, treeSize-1, treeSize-1)
+	if err != nil {
+		return nil, err
+	}
+	leafHash := merkletree.HashLeaf(entries[0].Leaf_input)
+	tree := new(merkletree.CollapsedTree)
+	if treeSize > 1 {
+		response, err := ctlog.getProofByHash(ctx, &leafHash, treeSize)
+		if err != nil {
+			return nil, err
+		}
+		if response.LeafIndex != treeSize-1 {
+			// This can happen if the leaf hash is present in the tree in more than one place. Unfortunately, we can't reconstruct when tree if this happens. Fortunately, this is really unlikely, and most logs support get-entry-and-proof anyways.
+			return nil, fmt.Errorf("unable to reconstruct tree because leaf hash %s is present in tree at more than one index (need proof for index %d but get-proof-by-hash returned proof for index %d)", leafHash.Base64String(), treeSize-1, response.LeafIndex)
+		}
+		slices.Reverse(response.AuditPath)
+		if err := tree.Init(response.AuditPath, treeSize-1); err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for hash %s to STH %d: %w", leafHash.Base64String(), treeSize, err)
+		}
+	}
+	tree.Add(leafHash)
+	return tree, nil
+}
+
+func (ctlog *RFC6962Log) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	tree, err := ctlog.reconstructTree(ctx, sth.TreeSize)
+	if err != nil {
+		return nil, err
+	}
+
+	if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash {
+		return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (entry *RFC6962LogEntry) isX509() bool {
+	return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 0
+}
+
+func (entry *RFC6962LogEntry) isPrecert() bool {
+	return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 1
+}
+
+func (entry *RFC6962LogEntry) LeafInput() []byte {
+	return entry.Leaf_input
+}
+
+func (entry *RFC6962LogEntry) ExtraData(context.Context, IssuerGetter) ([]byte, error) {
+	return entry.Extra_data, nil
+}
+
+func (entry *RFC6962LogEntry) Precertificate() (cttypes.ASN1Cert, error) {
+	if !entry.isPrecert() {
+		return nil, fmt.Errorf("not a precertificate entry")
+	}
+	extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing extra_data: %w", err)
+	}
+	return extraData.PreCertificate, nil
+}
+
+func (entry *RFC6962LogEntry) ChainFingerprints() ([][32]byte, error) {
+	chain, err := entry.parseChain()
+	if err != nil {
+		return nil, err
+	}
+	fingerprints := make([][32]byte, len(chain))
+	for i := range chain {
+		fingerprints[i] = sha256.Sum256(chain[i])
+	}
+	return fingerprints, nil
+}
+
+func (entry *RFC6962LogEntry) GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error) {
+	return entry.parseChain()
+}
+
+func (entry *RFC6962LogEntry) parseChain() (cttypes.ASN1CertChain, error) {
+	switch {
+	case entry.isX509():
+		extraData, err := cttypes.ParseExtraDataForX509Entry(entry.Extra_data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing extra_data for X509 entry: %w", err)
+		}
+		return extraData, nil
+	case entry.isPrecert():
+		extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing extra_data for precert entry: %w", err)
+		}
+		return extraData.PrecertificateChain, nil
+	default:
+		return nil, fmt.Errorf("unknown entry type")
+	}
+}

third_party/certspotter/ctclient/static.go

@@ -0,0 +1,401 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+	"strconv"
+	"sync"
+
+	"golang.org/x/crypto/cryptobyte"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+const (
+	staticTileHeight = 8
+	StaticTileWidth  = 1 << staticTileHeight
+)
+
+func staticSubtreeSize(level uint64) uint64 { return 1 << (level * staticTileHeight) }
+
+type StaticLog struct {
+	SubmissionURL *url.URL
+	MonitoringURL *url.URL
+	ID            cttypes.LogID
+	HTTPClient    *http.Client // nil to use default client
+}
+
+type StaticLogEntry struct {
+	timestampedEntry []byte
+	precertificate   []byte // nil iff x509 entry; non-nil iff precert entry
+	chain            [][32]byte
+}
+
+func (ctlog *StaticLog) AddChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.SubmissionURL, false, chain)
+}
+
+func (ctlog *StaticLog) AddPreChain(ctx context.Context, chain [][]byte) (*cttypes.SignedCertificateTimestamp, error) {
+	return addChainOrPreChain(ctx, ctlog.HTTPClient, ctlog.SubmissionURL, true, chain)
+}
+
+func (ctlog *StaticLog) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) {
+	fullURL := ctlog.MonitoringURL.JoinPath("/checkpoint").String()
+	responseBody, err := getBytes(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, fullURL, err
+	}
+	sth, err := cttypes.ParseCheckpoint(responseBody, ctlog.ID)
+	if err != nil {
+		return nil, fullURL, err
+	}
+	return sth, fullURL, nil
+}
+
+func (ctlog *StaticLog) GetRoots(ctx context.Context) ([][]byte, error) {
+	return getRoots(ctx, ctlog.HTTPClient, ctlog.SubmissionURL)
+}
+
+func (ctlog *StaticLog) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]StaticLogEntry, error) {
+	var (
+		tile       = startInclusive / StaticTileWidth
+		skip       = startInclusive % StaticTileWidth
+		tileWidth  = min(StaticTileWidth, endInclusive+1-tile*StaticTileWidth)
+		numEntries = tileWidth - skip
+	)
+
+	data, err := ctlog.getDataTile(ctx, tile, tileWidth)
+	if err != nil {
+		return nil, err
+	}
+	var skippedEntry StaticLogEntry
+	for i := range skip {
+		if rest, err := skippedEntry.parse(data); err != nil {
+			return nil, fmt.Errorf("error parsing skipped entry %d in tile %d: %w", i, tile, err)
+		} else {
+			data = rest
+		}
+	}
+	entries := make([]StaticLogEntry, numEntries)
+	for i := range numEntries {
+		if rest, err := entries[i].parse(data); err != nil {
+			return nil, fmt.Errorf("error parsing entry %d in tile %d: %w", skip+i, tile, err)
+		} else {
+			data = rest
+		}
+	}
+	return entries, nil
+}
+
+func (ctlog *StaticLog) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) {
+	nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive)
+	if err != nil {
+		return nil, err
+	}
+	entries := make([]Entry, len(nativeEntries))
+	for i := range nativeEntries {
+		entries[i] = &nativeEntries[i]
+	}
+	return entries, nil
+}
+
+func (ctlog *StaticLog) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	type job struct {
+		level  uint64
+		offset uint64
+		width  uint64
+	}
+	var jobs []job
+	for level, size := uint64(0), sth.TreeSize; size > 0; level++ {
+		fullTiles := size / StaticTileWidth
+		remainder := size % StaticTileWidth
+		size = fullTiles
+		if remainder > 0 {
+			jobs = append(jobs, job{
+				level:  level,
+				offset: fullTiles,
+				width:  remainder,
+			})
+		}
+	}
+	trees := make([]*merkletree.CollapsedTree, len(jobs))
+	errs := make([]error, len(jobs))
+
+	var wg sync.WaitGroup
+	for i, job := range jobs {
+		wg.Add(1)
+		go func() {
+			trees[i], errs[i] = ctlog.getTileCollapsedTree(ctx, job.level, job.offset, job.width)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	if err := errors.Join(errs...); err != nil {
+		return nil, err
+	}
+
+	tree := new(merkletree.CollapsedTree)
+	for _, subtree := range slices.Backward(trees) {
+		if err := tree.Append(*subtree); err != nil {
+			panic(err)
+		}
+	}
+
+	if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash {
+		return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (ctlog *StaticLog) getDataTile(ctx context.Context, tile uint64, width uint64) ([]byte, error) {
+	if width == 0 || width > StaticTileWidth {
+		panic("width is out of range")
+	}
+	var partialErr error
+	if width < StaticTileWidth {
+		fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, width)).String()
+		if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+			partialErr = err
+		} else {
+			return data, nil
+		}
+	}
+
+	fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, 0)).String()
+	if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+		if partialErr != nil {
+			return nil, partialErr
+		} else {
+			return nil, err
+		}
+	} else {
+		return data, nil
+	}
+}
+
+// returned slice is numHashes*merkletree.HashLen bytes long
+func (ctlog *StaticLog) getTile(ctx context.Context, level uint64, tile uint64, numHashes uint64) ([]byte, error) {
+	if numHashes == 0 || numHashes > StaticTileWidth {
+		panic("numHashes is out of range")
+	}
+
+	var partialErr error
+	if numHashes < StaticTileWidth {
+		fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, numHashes)).String()
+		if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+			partialErr = err
+		} else if expectedLen := merkletree.HashLen * int(numHashes); len(data) != expectedLen {
+			return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen)
+		} else {
+			return data, nil
+		}
+	}
+
+	fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, 0)).String()
+	if data, err := getBytes(ctx, ctlog.HTTPClient, fullURL); err != nil {
+		if partialErr != nil {
+			return nil, partialErr
+		} else {
+			return nil, err
+		}
+	} else if expectedLen := merkletree.HashLen * StaticTileWidth; len(data) != expectedLen {
+		return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen)
+	} else {
+		desiredLen := merkletree.HashLen * int(numHashes)
+		return data[:desiredLen], nil
+	}
+}
+
+func (ctlog *StaticLog) getTileCollapsedTree(ctx context.Context, level uint64, tile uint64, numHashes uint64) (*merkletree.CollapsedTree, error) {
+	data, err := ctlog.getTile(ctx, level, tile, numHashes)
+	if err != nil {
+		return nil, err
+	}
+	subtreeSize := staticSubtreeSize(level)
+	offset := staticSubtreeSize(level+1) * tile
+
+	tree := new(merkletree.CollapsedTree)
+	if err := tree.InitSubtree(offset, nil, 0); err != nil {
+		panic(err)
+	}
+	for i := uint64(0); i < numHashes; i++ {
+		hash := (merkletree.Hash)(data[i*merkletree.HashLen : (i+1)*merkletree.HashLen])
+		var subtree merkletree.CollapsedTree
+		if err := subtree.InitSubtree(offset+i*subtreeSize, []merkletree.Hash{hash}, subtreeSize); err != nil {
+			panic(err)
+		}
+		if err := tree.Append(subtree); err != nil {
+			panic(err)
+		}
+	}
+	return tree, nil
+}
+
+func (ctlog *StaticLog) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	fullURL := ctlog.MonitoringURL.JoinPath("/issuer/" + hex.EncodeToString(fingerprint[:])).String()
+	data, err := getBytes(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	if gotFingerprint := sha256.Sum256(data); gotFingerprint != *fingerprint {
+		return nil, fmt.Errorf("%s returned incorrect data with fingerprint %x", fullURL, gotFingerprint[:])
+	}
+	return data, nil
+}
+
+func (entry *StaticLogEntry) parse(input []byte) ([]byte, error) {
+	var skipped cryptobyte.String
+	str := cryptobyte.String(input)
+
+	// TimestampedEntry.timestamp
+	if !str.Skip(8) {
+		return nil, fmt.Errorf("error reading timestamp")
+	}
+	// TimestampedEntry.entry_type
+	var entryType uint16
+	if !str.ReadUint16(&entryType) {
+		return nil, fmt.Errorf("error reading entry type")
+	}
+	// TimestampedEntry.signed_entry
+	if entryType == 0 {
+		if !str.ReadUint24LengthPrefixed(&skipped) {
+			return nil, fmt.Errorf("error reading certificate")
+		}
+	} else if entryType == 1 {
+		if !str.Skip(32) {
+			return nil, fmt.Errorf("error reading issuer_key_hash")
+		}
+		if !str.ReadUint24LengthPrefixed(&skipped) {
+			return nil, fmt.Errorf("error reading tbs_certificate")
+		}
+	} else {
+		return nil, fmt.Errorf("invalid entry type %d", entryType)
+	}
+
+	// TimestampedEntry.extensions
+	if !str.ReadUint16LengthPrefixed(&skipped) {
+		return nil, fmt.Errorf("error reading extensions")
+	}
+
+	timestampedEntryLen := len(input) - len(str)
+	entry.timestampedEntry = input[:timestampedEntryLen]
+
+	// precertificate
+	if entryType == 1 {
+		var precertificate cryptobyte.String
+		if !str.ReadUint24LengthPrefixed(&precertificate) {
+			return nil, fmt.Errorf("error reading precertificate")
+		}
+		entry.precertificate = precertificate
+	} else {
+		entry.precertificate = nil
+	}
+
+	// certificate_chain
+	var chainBytes cryptobyte.String
+	if !str.ReadUint16LengthPrefixed(&chainBytes) {
+		return nil, fmt.Errorf("error reading certificate_chain")
+	}
+	entry.chain = make([][32]byte, 0, len(chainBytes)/32)
+	for !chainBytes.Empty() {
+		var fingerprint [32]byte
+		if !chainBytes.CopyBytes(fingerprint[:]) {
+			return nil, fmt.Errorf("error reading fingerprint in certificate_chain")
+		}
+		entry.chain = append(entry.chain, fingerprint)
+	}
+
+	return str, nil
+}
+
+func (entry *StaticLogEntry) LeafInput() []byte {
+	return append([]byte{0, 0}, entry.timestampedEntry...)
+}
+
+func (entry *StaticLogEntry) ExtraData(ctx context.Context, issuerGetter IssuerGetter) ([]byte, error) {
+	b := cryptobyte.NewBuilder(nil)
+	if entry.precertificate != nil {
+		b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(entry.precertificate)
+		})
+	}
+	b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+		for _, fingerprint := range entry.chain {
+			b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+				cert, err := issuerGetter.GetIssuer(ctx, &fingerprint)
+				if err != nil {
+					panic(cryptobyte.BuildError{Err: fmt.Errorf("error getting issuer %x: %w", fingerprint, err)})
+				}
+				b.AddBytes(cert)
+			})
+		}
+	})
+	return b.Bytes()
+}
+
+func (entry *StaticLogEntry) Precertificate() (cttypes.ASN1Cert, error) {
+	if entry.precertificate == nil {
+		return nil, fmt.Errorf("not a precertificate entry")
+	}
+	return entry.precertificate, nil
+}
+
+func (entry *StaticLogEntry) ChainFingerprints() ([][32]byte, error) {
+	return entry.chain, nil
+}
+
+func (entry *StaticLogEntry) GetChain(ctx context.Context, issuerGetter IssuerGetter) (cttypes.ASN1CertChain, error) {
+	var (
+		chain = make(cttypes.ASN1CertChain, len(entry.chain))
+		errs  = make([]error, len(entry.chain))
+	)
+	var wg sync.WaitGroup
+	for i, fingerprint := range entry.chain {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			chain[i], errs[i] = issuerGetter.GetIssuer(ctx, &fingerprint)
+		}()
+	}
+	wg.Wait()
+	if err := errors.Join(errs...); err != nil {
+		return nil, err
+	}
+	return chain, nil
+}
+
+func formatTilePath(level string, tile uint64, partial uint64) string {
+	path := "tile/" + level + "/" + formatTileIndex(tile)
+	if partial != 0 {
+		path += fmt.Sprintf(".p/%d", partial)
+	}
+	return path
+}
+
+func formatTileIndex(tile uint64) string {
+	const base = 1000
+	str := fmt.Sprintf("%03d", tile%base)
+	for tile >= base {
+		tile = tile / base
+		str = fmt.Sprintf("x%03d/%s", tile%base, str)
+	}
+	return str
+}

third_party/certspotter/ctclient/static_test.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"testing"
+)
+
+func TestFormatTileIndex(t *testing.T) {
+	tests := []struct {
+		in  uint64
+		out string
+	}{
+		{0, "000"},
+		{1, "001"},
+		{12, "012"},
+		{105, "105"},
+		{1000, "x001/000"},
+		{1050, "x001/050"},
+		{52123, "x052/123"},
+		{999001, "x999/001"},
+		{1999001, "x001/x999/001"},
+		{15999001, "x015/x999/001"},
+	}
+	for i, test := range tests {
+		result := formatTileIndex(test.in)
+		if result != test.out {
+			t.Errorf("#%d: formatTileIndex(%q) = %q, want %q", i, test.in, result, test.out)
+		}
+	}
+}