ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched)
CertSpotter internals, classifies cloud/database/storage DNS identifiers
in-process with a zero-allocation classifier, enriches via round-robin DNS
resolvers with CNAME following, and batch-writes useful findings to PostgreSQL.

- third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files
- internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI
- internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover
- internal/pipeline: bounded queues, dedup, direct-priority load shedding
- internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents)
- internal/probe: optional non-destructive reachability checks (off by default)
- Docker: distroless non-root, read-only rootfs, no app volume, resource limits
- tests enforce 0 allocs/op on the classifier and sink hot path

third_party/certspotter/loglist/load.go

@@ -0,0 +1,129 @@
+// Copyright (C) 2020, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+var UserAgent = ""
+
+// maxResponseBytes bounds the size of a log list we are willing to read into memory.
+const maxResponseBytes = 16 << 20 // 16 MiB
+
+// httpClient is used for fetching log lists.
+var httpClient = &http.Client{
+	Timeout: 60 * time.Second,
+}
+
+type ModificationToken struct {
+	etag     string
+	modified time.Time
+}
+
+var ErrNotModified = errors.New("loglist has not been modified")
+
+func newModificationToken(response *http.Response) *ModificationToken {
+	token := &ModificationToken{
+		etag: response.Header.Get("ETag"),
+	}
+	if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil {
+		token.modified = t
+	}
+	return token
+}
+
+func (token *ModificationToken) setRequestHeaders(request *http.Request) {
+	if token.etag != "" {
+		request.Header.Set("If-None-Match", token.etag)
+	} else if !token.modified.IsZero() {
+		request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat))
+	}
+}
+
+func Load(ctx context.Context, urlOrFile string) (*List, error) {
+	list, _, err := LoadIfModified(ctx, urlOrFile, nil)
+	return list, err
+}
+
+func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) {
+	if strings.HasPrefix(urlOrFile, "https://") {
+		return FetchIfModified(ctx, urlOrFile, token)
+	} else {
+		list, err := ReadFile(urlOrFile)
+		return list, nil, err
+	}
+}
+
+func Fetch(ctx context.Context, url string) (*List, error) {
+	list, _, err := FetchIfModified(ctx, url, nil)
+	return list, err
+}
+
+func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+	if token != nil {
+		token.setRequestHeaders(request)
+	}
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, nil, err
+	}
+	content, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
+	response.Body.Close()
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(content) > maxResponseBytes {
+		return nil, nil, fmt.Errorf("%s: response body exceeds maximum allowed size of %d bytes", url, maxResponseBytes)
+	}
+	if token != nil && response.StatusCode == http.StatusNotModified {
+		return nil, nil, ErrNotModified
+	}
+	if response.StatusCode != 200 {
+		return nil, nil, fmt.Errorf("%s: %s", url, response.Status)
+	}
+	list, err := Unmarshal(content)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing %s: %w", url, err)
+	}
+	return list, newModificationToken(response), err
+}
+
+func ReadFile(filename string) (*List, error) {
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return Unmarshal(content)
+}
+
+func Unmarshal(jsonBytes []byte) (*List, error) {
+	list := new(List)
+	if err := json.Unmarshal(jsonBytes, list); err != nil {
+		return nil, err
+	}
+	if err := list.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid log list: %s", err)
+	}
+	return list, nil
+}