ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
2026-09-22 13:13:10 +02:00
1 parent 9e78a114ac
commit e8f7d6bfdf
@@ -0,0 +1,182 @@
+// Copyright (C) 2022 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/bits"
+	"slices"
+)
+
+// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
+// nodes and calculating the root hash.
+type CollapsedTree struct {
+	offset uint64
+	nodes  []Hash
+	size   uint64
+}
+
+func calculateNumNodes(size uint64) int {
+	return bits.OnesCount64(size)
+}
+
+// TODO: phase out this function
+func EmptyCollapsedTree() *CollapsedTree {
+	return &CollapsedTree{nodes: []Hash{}, size: 0}
+}
+
+// TODO: phase out this function
+func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
+	tree := new(CollapsedTree)
+	if err := tree.Init(nodes, size); err != nil {
+		return nil, err
+	}
+	return tree, nil
+}
+
+func (tree CollapsedTree) Equal(other CollapsedTree) bool {
+	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
+}
+
+func (tree CollapsedTree) Clone() CollapsedTree {
+	return CollapsedTree{
+		offset: tree.offset,
+		nodes:  slices.Clone(tree.nodes),
+		size:   tree.size,
+	}
+}
+
+// Add a new leaf hash to the end of the tree.
+// Returns an error if and only if the new tree would be too large for the subtree offset.
+// Always returns a nil error if tree.Offset() == 0.
+func (tree *CollapsedTree) Add(hash Hash) error {
+	if tree.offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if tree.size+1 > maxSize {
+			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
+		}
+	}
+	tree.nodes = append(tree.nodes, hash)
+	tree.size++
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) Append(other CollapsedTree) error {
+	if tree.offset+tree.size != other.offset {
+		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
+	}
+	if tree.offset > 0 {
+		newSize := tree.size + other.size
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if newSize > maxSize {
+			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
+		}
+	}
+	if tree.size > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+		if other.size > maxSize {
+			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+		}
+	}
+
+	tree.nodes = append(tree.nodes, other.nodes...)
+	tree.size += other.size
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) collapse() {
+	numNodes := calculateNumNodes(tree.size)
+	for len(tree.nodes) > numNodes {
+		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
+		tree.nodes = tree.nodes[:len(tree.nodes)-2]
+		tree.nodes = append(tree.nodes, HashChildren(left, right))
+	}
+}
+
+func (tree CollapsedTree) CalculateRoot() Hash {
+	if len(tree.nodes) == 0 {
+		return HashNothing()
+	}
+	i := len(tree.nodes) - 1
+	hash := tree.nodes[i]
+	for i > 0 {
+		i -= 1
+		hash = HashChildren(tree.nodes[i], hash)
+	}
+	return hash
+}
+
+// Return the subtree offset (0 if this represents an entire tree)
+func (tree CollapsedTree) Offset() uint64 {
+	return tree.offset
+}
+
+// Return a non-nil slice containing the nodes.  The slice
+// must not be modified.
+func (tree CollapsedTree) Nodes() []Hash {
+	if tree.nodes == nil {
+		return []Hash{}
+	} else {
+		return tree.nodes
+	}
+}
+
+// Return the number of leaf nodes in the tree.
+func (tree CollapsedTree) Size() uint64 {
+	return tree.size
+}
+
+type collapsedTreeMessage struct {
+	Offset uint64 `json:"offset,omitzero"`
+	Nodes  []Hash `json:"nodes"` // never nil
+	Size   uint64 `json:"size"`
+}
+
+func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(collapsedTreeMessage{
+		Offset: tree.offset,
+		Nodes:  tree.Nodes(),
+		Size:   tree.size,
+	})
+}
+
+func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
+	var rawTree collapsedTreeMessage
+	if err := json.Unmarshal(b, &rawTree); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}
+
+func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
+	if len(nodes) != calculateNumNodes(size) {
+		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	}
+	tree.size = size
+	tree.nodes = nodes
+	return nil
+}
+
+func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
+	if offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(offset)
+		if size > maxSize {
+			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
+		}
+	}
+	tree.offset = offset
+	return tree.Init(nodes, size)
+}
@@ -0,0 +1,122 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+)
+
+// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
+type FragmentedCollapsedTree struct {
+	subtrees []CollapsedTree // sorted by offset
+}
+
+func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
+	return tree.Add(CollapsedTree{
+		offset: position,
+		nodes:  []Hash{hash},
+		size:   1,
+	})
+}
+
+func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
+	if subtree.size == 0 {
+		return nil
+	}
+	i := len(tree.subtrees)
+	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
+		i--
+	}
+	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
+		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
+	}
+	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
+		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
+	}
+	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
+		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
+		i++
+	}
+	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
+		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
+	}
+	return nil
+}
+
+func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
+	// TODO: try to make this linear time instead of quadratic; it should be possible since the subtrees are sorted by offset
+	for _, subtree := range other.subtrees {
+		if err := tree.Add(subtree); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
+	var prevEnd uint64
+	for i := range tree.subtrees {
+		if prevEnd != tree.subtrees[i].offset {
+			if !yield(prevEnd, tree.subtrees[i].offset) {
+				return
+			}
+		}
+		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
+	}
+	yield(prevEnd, 0)
+}
+
+func (tree FragmentedCollapsedTree) NumSubtrees() int {
+	return len(tree.subtrees)
+}
+
+func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
+	return tree.subtrees[i]
+}
+
+func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
+	if tree.subtrees == nil {
+		return []CollapsedTree{}
+	} else {
+		return tree.subtrees
+	}
+}
+
+// Return true iff the tree contains at least the first n nodes (without any gaps)
+func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
+	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
+}
+
+func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
+	for i := 1; i < len(subtrees); i++ {
+		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
+			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
+		}
+	}
+	tree.subtrees = subtrees
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(tree.Subtrees())
+}
+
+func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
+	var subtrees []CollapsedTree
+	if err := json.Unmarshal(b, &subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.Init(subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}
@@ -0,0 +1,77 @@
+// Copyright (C) 2022 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+)
+
+const HashLen = 32
+
+type Hash [HashLen]byte
+
+func (h Hash) Compare(other Hash) int {
+	return bytes.Compare(h[:], other[:])
+}
+
+func (h Hash) Base64String() string {
+	return base64.StdEncoding.EncodeToString(h[:])
+}
+
+func (h Hash) MarshalJSON() ([]byte, error) {
+	return json.Marshal(h[:])
+}
+
+func (h Hash) MarshalBinary() ([]byte, error) {
+	return h[:], nil
+}
+
+func (h *Hash) UnmarshalJSON(b []byte) error {
+	var hashBytes []byte
+	if err := json.Unmarshal(b, &hashBytes); err != nil {
+		return err
+	}
+	return h.UnmarshalBinary(hashBytes)
+}
+
+func (h *Hash) UnmarshalBinary(hashBytes []byte) error {
+	if len(hashBytes) != HashLen {
+		return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes))
+	}
+	copy(h[:], hashBytes)
+	return nil
+}
+
+func HashNothing() Hash {
+	return sha256.Sum256(nil)
+}
+
+func HashLeaf(leafBytes []byte) Hash {
+	var hash Hash
+	hasher := sha256.New()
+	hasher.Write([]byte{0x00})
+	hasher.Write(leafBytes)
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+func HashChildren(left Hash, right Hash) Hash {
+	var hash Hash
+	hasher := sha256.New()
+	hasher.Write([]byte{0x01})
+	hasher.Write(left[:])
+	hasher.Write(right[:])
+	hasher.Sum(hash[:0])
+	return hash
+}