ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

third_party/certspotter/loglist/schema.go

@@ -0,0 +1,158 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type List struct {
+	Version          string     `json:"version,omitzero"`
+	LogListTimestamp time.Time  `json:"log_list_timestamp,omitzero"` // Only present in v3 of schema
+	Operators        []Operator `json:"operators,omitzero"`
+}
+
+type Operator struct {
+	Name      string   `json:"name"`
+	Email     []string `json:"email,omitzero"`
+	Logs      []Log    `json:"logs,omitzero"`
+	TiledLogs []Log    `json:"tiled_logs,omitzero"`
+}
+
+// Log represents the log object in the "v3" JSON log list schema, as published by Chrome, Apple, and SSLMate.
+type Log struct {
+	Key              []byte        `json:"key"`
+	LogID            cttypes.LogID `json:"log_id"`
+	MMD              int           `json:"mmd"`
+	URL              string        `json:"url,omitzero"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitzero"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitzero"` // only for static-ct-api logs
+	Description      string        `json:"description"`
+	State            State         `json:"state,omitzero"`
+	LogType          LogType       `json:"log_type,omitzero"`
+	TemporalInterval *struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval,omitzero"`
+	LogSoftware *struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+	} `json:"log_software,omitzero"`
+
+	// certspotter-specific extensions
+	CertspotterDownloadSize int     `json:"certspotter_download_size,omitzero"`
+	CertspotterDownloadJobs int     `json:"certspotter_download_jobs,omitzero"`
+	CertspotterDownloadQPS  float64 `json:"certspotter_download_qps,omitzero"`
+
+	// TODO: add previous_operators
+}
+
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
+type State struct {
+	Pending *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"pending,omitzero"`
+
+	Qualified *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"qualified,omitzero"`
+
+	Usable *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"usable,omitzero"`
+
+	Readonly *struct {
+		Timestamp     time.Time `json:"timestamp"`
+		FinalTreeHead struct {
+			TreeSize       int64  `json:"tree_size"`
+			SHA256RootHash []byte `json:"sha256_root_hash"`
+		} `json:"final_tree_head"`
+	} `json:"readonly,omitzero"`
+
+	Retired *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"retired,omitzero"`
+
+	Rejected *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"rejected,omitzero"`
+}
+
+func (state *State) IsApproved() bool {
+	return state.Qualified != nil || state.Usable != nil || state.Readonly != nil
+}
+
+func (state *State) WasApprovedAt(t time.Time) bool {
+	return state.Retired != nil && t.Before(state.Retired.Timestamp)
+}
+
+type LogType string
+
+const (
+	LogTypeProd = "prod"
+	LogTypeTest = "test"
+)
+
+type LogSpec string
+
+const (
+	StaticCTAPI = "static-ct-api"
+	RFC6962     = "rfc6962"
+)
+
+// InclusionRequest represents the "v2" JSON log object, as published by log operators to request inclusion in browsers.
+type InclusionRequest struct {
+	Key          []byte        `json:"key"`
+	LogID        cttypes.LogID `json:"log_id"`
+	MMDSeconds   int           `json:"mmd_seconds"`
+	FriendlyName string        `json:"friendly_name"`
+	LogSpec      LogSpec       `json:"log_spec"`
+	Endpoint     struct {
+		URL string `json:"url"`
+	} `json:"endpoint,omitzero"`
+	SubmissionEndpoint struct {
+		URL string `json:"url"`
+	} `json:"submission_endpoint,omitzero"`
+	MonitoringEndpoint struct {
+		URL string `json:"url"`
+	} `json:"monitoring_endpoint,omitzero"`
+	TemporalInterval struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval"`
+	LogSoftware struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+	} `json:"log_software"`
+}