ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

third_party/certspotter/loglist/helpers.go

@@ -0,0 +1,40 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+)
+
+// Return all tiled and non-tiled logs from all operators
+func (list *List) AllLogs() []*Log {
+	logs := []*Log{}
+	for operator := range list.Operators {
+		for log := range list.Operators[operator].Logs {
+			logs = append(logs, &list.Operators[operator].Logs[log])
+		}
+		for log := range list.Operators[operator].TiledLogs {
+			logs = append(logs, &list.Operators[operator].TiledLogs[log])
+		}
+	}
+	return logs
+}
+
+func (log *Log) LogIDString() string {
+	return log.LogID.Base64String()
+}
+
+func (log *Log) AcceptsExpiration(expiration time.Time) bool {
+	return log.TemporalInterval == nil || withinInterval(expiration, log.TemporalInterval.StartInclusive, log.TemporalInterval.EndExclusive)
+}
+
+func withinInterval(expiration, startInclusive, endExclusive time.Time) bool {
+	return !expiration.Before(startInclusive) && expiration.Before(endExclusive)
+}

third_party/certspotter/loglist/load.go

@@ -0,0 +1,129 @@
+// Copyright (C) 2020, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+var UserAgent = ""
+
+// maxResponseBytes bounds the size of a log list we are willing to read into memory.
+const maxResponseBytes = 16 << 20 // 16 MiB
+
+// httpClient is used for fetching log lists.
+var httpClient = &http.Client{
+	Timeout: 60 * time.Second,
+}
+
+type ModificationToken struct {
+	etag     string
+	modified time.Time
+}
+
+var ErrNotModified = errors.New("loglist has not been modified")
+
+func newModificationToken(response *http.Response) *ModificationToken {
+	token := &ModificationToken{
+		etag: response.Header.Get("ETag"),
+	}
+	if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil {
+		token.modified = t
+	}
+	return token
+}
+
+func (token *ModificationToken) setRequestHeaders(request *http.Request) {
+	if token.etag != "" {
+		request.Header.Set("If-None-Match", token.etag)
+	} else if !token.modified.IsZero() {
+		request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat))
+	}
+}
+
+func Load(ctx context.Context, urlOrFile string) (*List, error) {
+	list, _, err := LoadIfModified(ctx, urlOrFile, nil)
+	return list, err
+}
+
+func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) {
+	if strings.HasPrefix(urlOrFile, "https://") {
+		return FetchIfModified(ctx, urlOrFile, token)
+	} else {
+		list, err := ReadFile(urlOrFile)
+		return list, nil, err
+	}
+}
+
+func Fetch(ctx context.Context, url string) (*List, error) {
+	list, _, err := FetchIfModified(ctx, url, nil)
+	return list, err
+}
+
+func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+	if token != nil {
+		token.setRequestHeaders(request)
+	}
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, nil, err
+	}
+	content, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
+	response.Body.Close()
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(content) > maxResponseBytes {
+		return nil, nil, fmt.Errorf("%s: response body exceeds maximum allowed size of %d bytes", url, maxResponseBytes)
+	}
+	if token != nil && response.StatusCode == http.StatusNotModified {
+		return nil, nil, ErrNotModified
+	}
+	if response.StatusCode != 200 {
+		return nil, nil, fmt.Errorf("%s: %s", url, response.Status)
+	}
+	list, err := Unmarshal(content)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing %s: %w", url, err)
+	}
+	return list, newModificationToken(response), err
+}
+
+func ReadFile(filename string) (*List, error) {
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return Unmarshal(content)
+}
+
+func Unmarshal(jsonBytes []byte) (*List, error) {
+	list := new(List)
+	if err := json.Unmarshal(jsonBytes, list); err != nil {
+		return nil, err
+	}
+	if err := list.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid log list: %s", err)
+	}
+	return list, nil
+}

third_party/certspotter/loglist/mozilla/ctlogs.go

@@ -0,0 +1,275 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+// Package mozilla contains a parser for Mozilla's CTKnownLogs.h file
+package mozilla
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// CTLogInfo describes a certificate transparency log from Mozilla's CTKnownLogs.h
+// file.
+type CTLogInfo struct {
+	Name          string
+	State         string // Admissible or Retired
+	Protocol      string // RFC6962 or Tiled
+	Timestamp     time.Time
+	OperatorIndex int
+	Key           []byte
+}
+
+// CTLogOperatorInfo describes a CT log operator from Mozilla's CTKnownLogs.h
+// file.
+type CTLogOperatorInfo struct {
+	Name string
+	ID   int
+}
+
+// Parse reads the CTKnownLogs.h content from r and returns the parsed logs and
+// operators. Blocks enclosed by `#ifdef DEBUG` and `#endif` are ignored.
+func Parse(r io.Reader) ([]CTLogInfo, []CTLogOperatorInfo, error) {
+	scanner := bufio.NewScanner(r)
+	skip := 0
+	inLogs := false
+	inOps := false
+
+	var logs []CTLogInfo
+	var ops []CTLogOperatorInfo
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		trimmed := strings.TrimSpace(line)
+
+		switch {
+		case strings.HasPrefix(trimmed, "#ifdef DEBUG"):
+			skip++
+			continue
+		case strings.HasPrefix(trimmed, "#endif"):
+			if skip > 0 {
+				skip--
+				continue
+			}
+		}
+
+		if skip > 0 {
+			continue
+		}
+
+		if strings.HasPrefix(trimmed, "const CTLogInfo kCTLogList[]") {
+			inLogs = true
+			continue
+		}
+		if strings.HasPrefix(trimmed, "const CTLogOperatorInfo kCTLogOperatorList[]") {
+			inOps = true
+			continue
+		}
+
+		if inLogs {
+			if trimmed == "};" {
+				inLogs = false
+				continue
+			}
+			if strings.HasPrefix(trimmed, "{") {
+				log, err := readLogEntry(trimmed, scanner)
+				if err != nil {
+					return nil, nil, err
+				}
+				logs = append(logs, log)
+			}
+			continue
+		}
+
+		if inOps {
+			if trimmed == "};" {
+				inOps = false
+				continue
+			}
+			if strings.HasPrefix(trimmed, "{") {
+				op, err := readOperatorEntry(trimmed)
+				if err != nil {
+					return nil, nil, err
+				}
+				ops = append(ops, op)
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, nil, err
+	}
+
+	return logs, ops, nil
+}
+
+func readLogEntry(firstLine string, s *bufio.Scanner) (CTLogInfo, error) {
+	var log CTLogInfo
+
+	// Example first line:
+	// {"Name", CTLogState::Admissible, CTLogFormat::RFC6962,
+	firstLine = strings.TrimSpace(firstLine)
+	if !strings.HasPrefix(firstLine, "{") {
+		return log, errors.New("invalid log entry start")
+	}
+	firstLine = strings.TrimPrefix(firstLine, "{")
+	firstLine = strings.TrimSuffix(firstLine, ",")
+
+	parts := splitCSV(firstLine)
+
+	for len(parts) < 3 {
+		if !s.Scan() {
+			return log, io.ErrUnexpectedEOF
+		}
+		next := strings.TrimSpace(strings.TrimSuffix(s.Text(), ","))
+		parts = append(parts, splitCSV(next)...)
+	}
+
+	log.Name = trimQuotes(parts[0])
+	log.State = strings.TrimPrefix(strings.TrimSpace(parts[1]), "CTLogState::")
+	log.Protocol = strings.TrimPrefix(strings.TrimSpace(parts[2]), "CTLogFormat::")
+
+	// Next line: timestamp
+	if !s.Scan() {
+		return log, io.ErrUnexpectedEOF
+	}
+	tsLine := strings.TrimSpace(s.Text())
+	tsValue := strings.Split(tsLine, ",")[0]
+	ts, err := strconv.ParseInt(strings.TrimSpace(tsValue), 10, 64)
+	if err != nil {
+		return log, err
+	}
+	log.Timestamp = time.Unix(0, ts*int64(time.Millisecond))
+
+	// Next line: operator index
+	if !s.Scan() {
+		return log, io.ErrUnexpectedEOF
+	}
+	opLine := strings.TrimSpace(s.Text())
+	opValue := strings.Split(opLine, ",")[0]
+	opIndex, err := strconv.Atoi(strings.TrimSpace(opValue))
+	if err != nil {
+		return log, err
+	}
+	log.OperatorIndex = opIndex
+
+	// Key lines
+	var keyHex strings.Builder
+	for {
+		if !s.Scan() {
+			return log, io.ErrUnexpectedEOF
+		}
+		l := strings.TrimSpace(s.Text())
+		if strings.HasPrefix(l, "\"") {
+			// remove trailing comma if any
+			trimmed := strings.TrimSuffix(l, ",")
+			keyHex.WriteString(trimQuotes(trimmed))
+			if strings.HasSuffix(l, ",") {
+				// last key line
+				break
+			}
+			continue
+		}
+		return log, errors.New("unexpected line while reading key")
+	}
+
+	key, err := decodeHexEscapes(keyHex.String())
+	if err != nil {
+		return log, err
+	}
+
+	// key length line
+	if !s.Scan() {
+		return log, io.ErrUnexpectedEOF
+	}
+	lenLine := strings.TrimSpace(s.Text())
+	lenValue := strings.TrimSuffix(lenLine, "},")
+	keyLen, err := strconv.Atoi(strings.TrimSpace(lenValue))
+	if err != nil {
+		return log, err
+	}
+	if len(key) != keyLen {
+		// ignore mismatch but continue
+	}
+	log.Key = key
+	return log, nil
+}
+
+func readOperatorEntry(line string) (CTLogOperatorInfo, error) {
+	var op CTLogOperatorInfo
+	line = strings.TrimSuffix(strings.TrimSpace(line), ",")
+	if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") {
+		return op, errors.New("invalid operator entry")
+	}
+	line = strings.TrimPrefix(line, "{")
+	line = strings.TrimSuffix(line, "}")
+	parts := splitCSV(line)
+	if len(parts) != 2 {
+		return op, errors.New("invalid operator fields")
+	}
+	op.Name = trimQuotes(parts[0])
+	id, err := strconv.Atoi(strings.TrimSpace(parts[1]))
+	if err != nil {
+		return op, err
+	}
+	op.ID = id
+	return op, nil
+}
+
+func splitCSV(s string) []string {
+	var parts []string
+	var cur strings.Builder
+	inQuote := false
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if c == '"' {
+			inQuote = !inQuote
+			cur.WriteByte(c)
+			continue
+		}
+		if c == ',' && !inQuote {
+			parts = append(parts, strings.TrimSpace(cur.String()))
+			cur.Reset()
+			continue
+		}
+		cur.WriteByte(c)
+	}
+	if cur.Len() > 0 {
+		parts = append(parts, strings.TrimSpace(cur.String()))
+	}
+	return parts
+}
+
+func trimQuotes(s string) string {
+	s = strings.TrimSpace(s)
+	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
+		return s[1 : len(s)-1]
+	}
+	return s
+}
+
+func decodeHexEscapes(s string) ([]byte, error) {
+	var out []byte
+	for i := 0; i < len(s); {
+		if i+3 >= len(s) || s[i] != '\\' || s[i+1] != 'x' {
+			return nil, errors.New("invalid escape")
+		}
+		b, err := strconv.ParseUint(s[i+2:i+4], 16, 8)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, byte(b))
+		i += 4
+	}
+	return out, nil
+}

third_party/certspotter/loglist/mozilla/ctlogs_test.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package mozilla
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"testing"
+)
+
+// parseFromURL downloads the CTKnownLogs.h file from the given URL and parses it.
+func parseFromURL(url string) ([]CTLogInfo, []CTLogOperatorInfo, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, nil, errors.New(resp.Status)
+	}
+	return Parse(resp.Body)
+}
+
+func TestParseFromURL(t *testing.T) {
+	logs, ops, err := parseFromURL("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ops) == 0 {
+		t.Fatal("no operators parsed")
+	}
+	foundGoogle := false
+	foundSectigo := false
+	foundLets := false
+	for _, op := range ops {
+		if op.Name == "" {
+			t.Error("operator with empty name")
+		}
+		switch op.Name {
+		case "Google":
+			foundGoogle = true
+		case "Sectigo":
+			foundSectigo = true
+		case "Let's Encrypt":
+			foundLets = true
+		}
+	}
+	if !foundGoogle || !foundSectigo || !foundLets {
+		t.Errorf("missing expected operators: Google=%v Sectigo=%v Let's=%v", foundGoogle, foundSectigo, foundLets)
+	}
+
+	if len(logs) == 0 {
+		t.Fatal("no logs parsed")
+	}
+	foundHash := false
+	targetHash := "1219ENGn9XfCx+lf1wC/+YLJM1pl4dCzAXMXwMjFaXc="
+	for _, l := range logs {
+		if l.Name == "" {
+			t.Error("log with empty name")
+		}
+		if len(l.Key) == 0 {
+			t.Error("log with empty key")
+		}
+		if l.State != "Admissible" && l.State != "Retired" {
+			t.Errorf("unexpected state %q", l.State)
+		}
+		if l.Protocol != "RFC6962" && l.Protocol != "Tiled" {
+			t.Errorf("unexpected protocol %q", l.Protocol)
+		}
+		hash := sha256.Sum256(l.Key)
+		if base64.StdEncoding.EncodeToString(hash[:]) == targetHash {
+			foundHash = true
+		}
+	}
+	if !foundHash {
+		t.Errorf("log with key hash %s not found", targetHash)
+	}
+}

third_party/certspotter/loglist/mozilla/example_test.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package mozilla_test
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"software.sslmate.com/src/certspotter/loglist/mozilla"
+)
+
+func ExampleParse() {
+	resp, err := http.Get("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Fatal(resp.Status)
+	}
+	logs, operators, err := mozilla.Parse(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0)
+	fmt.Fprintln(tw, "Operator\tName")
+	for _, o := range operators {
+		fmt.Fprintf(tw, "%d\t%s\n", o.ID, o.Name)
+	}
+	tw.Flush()
+
+	tw = tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0)
+	fmt.Fprintln(tw, "LogID\tProtocol\tState\tTimestamp\tOperator\tName")
+	for _, l := range logs {
+		hash := sha256.Sum256(l.Key)
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%s\n",
+			base64.StdEncoding.EncodeToString(hash[:]),
+			l.Protocol, l.State, l.Timestamp.UTC().Format(time.RFC3339), l.OperatorIndex, l.Name)
+	}
+	tw.Flush()
+}

third_party/certspotter/loglist/schema.go

@@ -0,0 +1,158 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type List struct {
+	Version          string     `json:"version,omitzero"`
+	LogListTimestamp time.Time  `json:"log_list_timestamp,omitzero"` // Only present in v3 of schema
+	Operators        []Operator `json:"operators,omitzero"`
+}
+
+type Operator struct {
+	Name      string   `json:"name"`
+	Email     []string `json:"email,omitzero"`
+	Logs      []Log    `json:"logs,omitzero"`
+	TiledLogs []Log    `json:"tiled_logs,omitzero"`
+}
+
+// Log represents the log object in the "v3" JSON log list schema, as published by Chrome, Apple, and SSLMate.
+type Log struct {
+	Key              []byte        `json:"key"`
+	LogID            cttypes.LogID `json:"log_id"`
+	MMD              int           `json:"mmd"`
+	URL              string        `json:"url,omitzero"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitzero"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitzero"` // only for static-ct-api logs
+	Description      string        `json:"description"`
+	State            State         `json:"state,omitzero"`
+	LogType          LogType       `json:"log_type,omitzero"`
+	TemporalInterval *struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval,omitzero"`
+	LogSoftware *struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+	} `json:"log_software,omitzero"`
+
+	// certspotter-specific extensions
+	CertspotterDownloadSize int     `json:"certspotter_download_size,omitzero"`
+	CertspotterDownloadJobs int     `json:"certspotter_download_jobs,omitzero"`
+	CertspotterDownloadQPS  float64 `json:"certspotter_download_qps,omitzero"`
+
+	// TODO: add previous_operators
+}
+
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
+type State struct {
+	Pending *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"pending,omitzero"`
+
+	Qualified *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"qualified,omitzero"`
+
+	Usable *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"usable,omitzero"`
+
+	Readonly *struct {
+		Timestamp     time.Time `json:"timestamp"`
+		FinalTreeHead struct {
+			TreeSize       int64  `json:"tree_size"`
+			SHA256RootHash []byte `json:"sha256_root_hash"`
+		} `json:"final_tree_head"`
+	} `json:"readonly,omitzero"`
+
+	Retired *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"retired,omitzero"`
+
+	Rejected *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"rejected,omitzero"`
+}
+
+func (state *State) IsApproved() bool {
+	return state.Qualified != nil || state.Usable != nil || state.Readonly != nil
+}
+
+func (state *State) WasApprovedAt(t time.Time) bool {
+	return state.Retired != nil && t.Before(state.Retired.Timestamp)
+}
+
+type LogType string
+
+const (
+	LogTypeProd = "prod"
+	LogTypeTest = "test"
+)
+
+type LogSpec string
+
+const (
+	StaticCTAPI = "static-ct-api"
+	RFC6962     = "rfc6962"
+)
+
+// InclusionRequest represents the "v2" JSON log object, as published by log operators to request inclusion in browsers.
+type InclusionRequest struct {
+	Key          []byte        `json:"key"`
+	LogID        cttypes.LogID `json:"log_id"`
+	MMDSeconds   int           `json:"mmd_seconds"`
+	FriendlyName string        `json:"friendly_name"`
+	LogSpec      LogSpec       `json:"log_spec"`
+	Endpoint     struct {
+		URL string `json:"url"`
+	} `json:"endpoint,omitzero"`
+	SubmissionEndpoint struct {
+		URL string `json:"url"`
+	} `json:"submission_endpoint,omitzero"`
+	MonitoringEndpoint struct {
+		URL string `json:"url"`
+	} `json:"monitoring_endpoint,omitzero"`
+	TemporalInterval struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval"`
+	LogSoftware struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+	} `json:"log_software"`
+}

third_party/certspotter/loglist/validate.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+func (list *List) Validate() error {
+	for i := range list.Operators {
+		if err := list.Operators[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth operator (%s): %w", i, list.Operators[i].Name, err)
+		}
+	}
+	return nil
+}
+
+func (operator *Operator) Validate() error {
+	for i := range operator.Logs {
+		if err := operator.Logs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+		}
+	}
+	for i := range operator.TiledLogs {
+		if err := operator.TiledLogs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
+		}
+	}
+	return nil
+}
+
+func (log *Log) Validate() error {
+	realLogID := sha256.Sum256(log.Key)
+	if log.LogID != realLogID {
+		return fmt.Errorf("log ID does not match log key")
+	}
+
+	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
+		return fmt.Errorf("URL(s) not provided")
+	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
+		return fmt.Errorf("inconsistent URLs provided")
+	}
+
+	if log.MMD < 0 {
+		return fmt.Errorf("log has a negative MMD")
+	}
+
+	return nil
+}