ctcloud: CertSpotter-backed CT cloud-resource monitor

Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
2026-09-22 13:13:10 +02:00
1 parent 9e78a114ac
commit e8f7d6bfdf
@@ -0,0 +1,39 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+)
+
+// IdentifierSink receives the DNS identifiers parsed from every monitored
+// certificate.  This is a CTCLOUD PATCH to upstream certspotter; see
+// CTCLOUD_PATCH.md.  When non-nil it is invoked on the monitoring hot path
+// instead of the watchlist/notification machinery.
+type IdentifierSink interface {
+	HandleIdentifiers(ctx context.Context, identifiers *certspotter.Identifiers)
+}
+
+type Config struct {
+	LogListSource       string
+	State               StateProvider
+	StartAtEnd          bool
+	WatchList           WatchList
+	Verbose             bool
+	HealthCheckInterval time.Duration
+
+	// IdentifierSink is a CTCLOUD PATCH (see CTCLOUD_PATCH.md).  When set,
+	// parsed identifiers are handed directly to it in-process and the normal
+	// watchlist/chain/notification path is skipped.
+	IdentifierSink IdentifierSink
+}
@@ -0,0 +1,173 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"golang.org/x/sync/errgroup"
+	"log"
+	insecurerand "math/rand"
+	"software.sslmate.com/src/certspotter/loglist"
+	"time"
+)
+
+const (
+	reloadLogListIntervalMin = 30 * time.Minute
+	reloadLogListIntervalMax = 90 * time.Minute
+)
+
+func randomDuration(min, max time.Duration) time.Duration {
+	return min + time.Duration(insecurerand.Int63n(int64(max-min+1)))
+}
+
+func reloadLogListInterval() time.Duration {
+	return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax)
+}
+
+type task struct {
+	log  *loglist.Log
+	stop context.CancelFunc
+}
+
+type daemon struct {
+	config         *Config
+	taskgroup      *errgroup.Group
+	tasks          map[LogID]task
+	logsLoadedAt   time.Time
+	logListToken   *loglist.ModificationToken
+}
+
+func (daemon *daemon) healthCheck(ctx context.Context) error {
+	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
+		errors, err := daemon.config.State.GetErrors(ctx, nil, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
+		var errorsDir string
+		if fsstate, ok := daemon.config.State.(*FilesystemState); ok {
+			errorsDir = fsstate.errorDir(nil)
+		}
+		info := &StaleLogListInfo{
+			Source:        daemon.config.LogListSource,
+			LastSuccess:   daemon.logsLoadedAt,
+			RecentErrors:  errors,
+			ErrorsDir:     errorsDir,
+		}
+		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
+			return fmt.Errorf("error notifying about stale log list: %w", err)
+		}
+	}
+
+	for _, task := range daemon.tasks {
+		if err := healthCheckLog(ctx, daemon.config, task.log); err != nil {
+			return fmt.Errorf("error checking health of log %q: %w", task.log.GetMonitoringURL(), err)
+		}
+	}
+	return nil
+}
+
+func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
+	ctx, cancel := context.WithCancel(ctx)
+	daemon.taskgroup.Go(func() error {
+		defer cancel()
+		err := monitorLogContinously(ctx, daemon.config, ctlog)
+		if daemon.config.Verbose {
+			log.Printf("%s: task stopped with error: %s", ctlog.GetMonitoringURL(), err)
+		}
+		if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
+			return nil
+		} else {
+			return fmt.Errorf("error while monitoring %s: %w", ctlog.GetMonitoringURL(), err)
+		}
+	})
+	return task{log: ctlog, stop: cancel}
+}
+
+func (daemon *daemon) loadLogList(ctx context.Context) error {
+	newLogList, newToken, err := getLogList(ctx, daemon.config.LogListSource, daemon.logListToken)
+	if errors.Is(err, loglist.ErrNotModified) {
+		daemon.logsLoadedAt = time.Now() // so the log list is not reported as stale
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if daemon.config.Verbose {
+		log.Printf("fetched %d logs from %q", len(newLogList), daemon.config.LogListSource)
+	}
+
+	for logID, task := range daemon.tasks {
+		if _, exists := newLogList[logID]; exists {
+			continue
+		}
+		if daemon.config.Verbose {
+			log.Printf("stopping task for log %s", logID.Base64String())
+		}
+		task.stop()
+		delete(daemon.tasks, logID)
+	}
+	for logID, ctlog := range newLogList {
+		if _, isRunning := daemon.tasks[logID]; isRunning {
+			continue
+		}
+		if daemon.config.Verbose {
+			log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.GetMonitoringURL())
+		}
+		daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
+	}
+	daemon.logsLoadedAt = time.Now()
+	daemon.logListToken = newToken
+	return nil
+}
+
+func (daemon *daemon) run(ctx context.Context) error {
+	if err := daemon.config.State.Prepare(ctx); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
+	}
+
+	if err := daemon.loadLogList(ctx); err != nil {
+		return fmt.Errorf("error loading log list: %w", err)
+	}
+
+	reloadLogListTicker := time.NewTicker(reloadLogListInterval())
+	defer reloadLogListTicker.Stop()
+
+	healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval)
+	defer healthCheckTicker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-reloadLogListTicker.C:
+			if err := daemon.loadLogList(ctx); err != nil {
+				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
+			}
+			reloadLogListTicker.Reset(reloadLogListInterval())
+		case <-healthCheckTicker.C:
+			if err := daemon.healthCheck(ctx); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func Run(ctx context.Context, config *Config) error {
+	group, ctx := errgroup.WithContext(ctx)
+	daemon := &daemon{
+		config:    config,
+		taskgroup: group,
+		tasks:     make(map[LogID]task),
+	}
+	group.Go(func() error { return daemon.run(ctx) })
+	return group.Wait()
+}
@@ -0,0 +1,189 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/pem"
+	"fmt"
+	"strings"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type DiscoveredCert struct {
+	WatchItem    WatchItem
+	LogEntry     *LogEntry
+	Info         *certspotter.CertInfo
+	Chain        []cttypes.ASN1Cert // first entry is the leaf certificate or precertificate
+	ChainError   error              // any error generating or validating Chain; if non-nil, Chain may be partial or incorrect
+	TBSSHA256    [32]byte           // computed over Info.TBS.Raw
+	SHA256       [32]byte           // computed over Chain[0]
+	PubkeySHA256 [32]byte           // computed over Info.TBS.PublicKey.FullBytes
+	Identifiers  *certspotter.Identifiers
+}
+
+type certPaths struct {
+	temporary bool
+
+	certPath string
+	jsonPath string
+	textPath string
+}
+
+func (cert *DiscoveredCert) pemChain() []byte {
+	var buffer bytes.Buffer
+	if cert.ChainError != nil {
+		fmt.Fprintln(&buffer, "Warning: this chain may be incomplete or invalid: ", cert.ChainError)
+	}
+	for _, certBytes := range cert.Chain {
+		if err := pem.Encode(&buffer, &pem.Block{
+			Type:  "CERTIFICATE",
+			Bytes: certBytes,
+		}); err != nil {
+			panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err))
+		}
+	}
+	return buffer.Bytes()
+}
+
+func (cert *DiscoveredCert) json() any {
+	object := map[string]any{
+		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
+		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
+		"dns_names":     cert.Identifiers.DNSNames,
+		"ip_addresses":  cert.Identifiers.IPAddrs,
+	}
+
+	if cert.Info.ValidityParseError == nil {
+		object["not_before"] = cert.Info.Validity.NotBefore
+		object["not_after"] = cert.Info.Validity.NotAfter
+	} else {
+		object["not_before"] = nil
+		object["not_after"] = nil
+	}
+
+	return object
+}
+
+func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
+	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
+		return err
+	}
+	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
+		return err
+	}
+	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
+		return err
+	}
+	return nil
+}
+
+func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
+	env := []string{
+		"EVENT=discovered_cert",
+		"SUMMARY=" + certNotificationSummary(cert),
+		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
+		"LOG_URI=" + cert.LogEntry.Log.GetMonitoringURL(),
+		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
+		"WATCH_ITEM=" + cert.WatchItem.String(),
+		"TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]),
+		"CERT_SHA256=" + hex.EncodeToString(cert.SHA256[:]),
+		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
+		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
+		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
+	}
+
+	if paths != nil {
+		env = append(env, "CERT_FILENAME="+paths.certPath)
+		env = append(env, "JSON_FILENAME="+paths.jsonPath)
+		env = append(env, "TEXT_FILENAME="+paths.textPath)
+	}
+
+	if cert.Info.ValidityParseError == nil {
+		env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String())
+		env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix()))
+		env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339))
+		env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String())
+		env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix()))
+		env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339))
+	} else {
+		env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error())
+	}
+
+	if cert.Info.SubjectParseError == nil {
+		env = append(env, "SUBJECT_DN="+cert.Info.Subject.String())
+	} else {
+		env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error())
+	}
+
+	if cert.Info.IssuerParseError == nil {
+		env = append(env, "ISSUER_DN="+cert.Info.Issuer.String())
+	} else {
+		env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error())
+	}
+
+	if cert.Info.SerialNumberParseError == nil {
+		env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber))
+	} else {
+		env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
+	}
+
+	if cert.ChainError != nil {
+		env = append(env, "CHAIN_ERROR="+cert.ChainError.Error())
+	}
+
+	return env
+}
+
+func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
+	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+
+	fmt.Fprintf(text, "%x:\n", cert.SHA256)
+	for _, dnsName := range cert.Identifiers.DNSNames {
+		writeField("DNS Name", dnsName)
+	}
+	for _, ipaddr := range cert.Identifiers.IPAddrs {
+		writeField("IP Address", ipaddr)
+	}
+	writeField("Pubkey", hex.EncodeToString(cert.PubkeySHA256[:]))
+	if cert.Info.IssuerParseError == nil {
+		writeField("Issuer", cert.Info.Issuer)
+	} else {
+		writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError))
+	}
+	if cert.Info.ValidityParseError == nil {
+		writeField("Not Before", cert.Info.Validity.NotBefore)
+		writeField("Not After", cert.Info.Validity.NotAfter)
+	} else {
+		writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
+		writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
+	}
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.GetMonitoringURL()))
+	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
+	if cert.ChainError != nil {
+		writeField("Error Building Chain", cert.ChainError.Error())
+	}
+	if paths != nil && !paths.temporary {
+		writeField("Filename", paths.certPath)
+	}
+
+	return text.String()
+}
+
+func certNotificationSummary(cert *DiscoveredCert) string {
+	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
+}
@@ -0,0 +1,28 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"log"
+
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
+	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
+		log.Print("unable to notify about error: ", err)
+		if ctlog == nil {
+			log.Print(errToRecord)
+		} else {
+			log.Print(ctlog.GetMonitoringURL(), ": ", errToRecord)
+		}
+	}
+}
@@ -0,0 +1,138 @@
+// Copyright (C) 2017, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"slices"
+	"strings"
+)
+
+func randomFileSuffix() string {
+	var randomBytes [12]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:])
+}
+
+func writeSyncFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	if err2 := f.Sync(); err2 != nil && err == nil {
+		err = err2
+	}
+	if err2 := f.Close(); err2 != nil && err == nil {
+		err = err2
+	}
+	return err
+}
+
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	tempname := filename + ".tmp." + randomFileSuffix()
+	if err := writeSyncFile(tempname, data, perm); err != nil {
+		return fmt.Errorf("error writing %s: %w", filename, err)
+	}
+	if err := os.Rename(tempname, filename); err != nil {
+		os.Remove(tempname)
+		return fmt.Errorf("error writing %s: %w", filename, err)
+	}
+	return nil
+}
+
+func writeTextFile(filename string, fileText string, perm os.FileMode) error {
+	return writeFile(filename, []byte(fileText), perm)
+}
+
+func writeJSONFile(filename string, data any, perm os.FileMode) error {
+	fileBytes, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	fileBytes = append(fileBytes, '\n')
+	return writeFile(filename, fileBytes, perm)
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}
+
+// countFiles returns the number of non-hidden files in dirPath whose
+// name ends in suffix.  Returns 0 if dirPath does not exist.
+func countFiles(dirPath string, suffix string) (int, error) {
+	entries, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return 0, nil
+	} else if err != nil {
+		return 0, err
+	}
+	count := 0
+	for _, entry := range entries {
+		if !entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") && strings.HasSuffix(entry.Name(), suffix) {
+			count++
+		}
+	}
+	return count, nil
+}
+
+func tailFile(filename string, linesWanted int) ([]byte, int, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer file.Close()
+	return tail(file, linesWanted, 4096)
+}
+
+func tail(r io.ReadSeeker, linesWanted int, chunkSize int) ([]byte, int, error) {
+	var buf []byte
+	linesGot := 0
+
+	offset, err := r.Seek(0, io.SeekEnd)
+	if err != nil {
+		return nil, 0, err
+	}
+	for offset > 0 {
+		readSize := chunkSize
+		if offset < int64(readSize) {
+			readSize = int(offset)
+		}
+		offset -= int64(readSize)
+		if _, err := r.Seek(offset, io.SeekStart); err != nil {
+			return nil, 0, err
+		}
+		buf = slices.Grow(buf, readSize)
+		copy(buf[readSize:len(buf)+readSize], buf)
+		buf = buf[:len(buf)+readSize]
+		if _, err := io.ReadFull(r, buf[:readSize]); err != nil {
+			return nil, 0, err
+		}
+		for i := readSize; i > 0; i-- {
+			if buf[i-1] == '\n' {
+				if linesGot == linesWanted {
+					return buf[i:], linesGot, nil
+				}
+				linesGot++
+			}
+		}
+	}
+	return buf, linesGot, nil
+}
@@ -0,0 +1,416 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+const keepErrorDays = 7
+const errorDateFormat = "2006-01-02"
+
+type FilesystemState struct {
+	StateDir  string
+	CacheDir  string
+	SaveCerts bool
+	Script    string
+	ScriptDir string
+	Email     []string
+	Stdout    bool
+	errorMu   sync.Mutex
+}
+
+func (s *FilesystemState) logStateDir(logID LogID) string {
+	return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
+}
+
+func (s *FilesystemState) Prepare(ctx context.Context) error {
+	if err := prepareStateDir(s.StateDir); err != nil {
+		return err
+	}
+	if err := prepareCacheDir(s.CacheDir); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
+	var (
+		stateDirPath        = s.logStateDir(logID)
+		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
+		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
+		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+		errorsDirPath       = filepath.Join(stateDirPath, "errors")
+	)
+	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath, errorsDirPath} {
+		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ListLogs returns the IDs of the logs that have a state directory (i.e., for
+// which PrepareLog has been called at some point).  Note that LoadLogState can
+// still return nil for a returned log ID, if no state has been stored yet.
+func (s *FilesystemState) ListLogs(ctx context.Context) ([]LogID, error) {
+	entries, err := os.ReadDir(filepath.Join(s.StateDir, "logs"))
+	if errors.Is(err, fs.ErrNotExist) {
+		return []LogID{}, nil
+	} else if err != nil {
+		return nil, err
+	}
+	logIDs := make([]LogID, 0, len(entries))
+	for _, entry := range entries {
+		name := entry.Name()
+		if strings.HasPrefix(name, ".") || !entry.IsDir() {
+			continue
+		}
+		nameBytes, err := base64.RawURLEncoding.DecodeString(name)
+		if err != nil || len(nameBytes) != 32 {
+			continue
+		}
+		var logID LogID
+		copy(logID[:], nameBytes)
+		logIDs = append(logIDs, logID)
+	}
+	return logIDs, nil
+}
+
+func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	fileBytes, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	state := new(LogState)
+	if err := json.Unmarshal(fileBytes, state); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return state, nil
+}
+
+func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	return writeJSONFile(filePath, state, 0666)
+}
+
+func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) (*StoredSTH, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return storeSTHInDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*StoredSTH, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return loadSTHsFromDir(sthsDirPath)
+}
+
+func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return removeSTHFromDir(sthsDirPath, sth)
+}
+
+// LoadLargestSTH returns the stored STH with the largest tree size, breaking
+// ties by later timestamp.  Returns nil if no STHs are stored for the log.
+func (s *FilesystemState) LoadLargestSTH(ctx context.Context, logID LogID) (*StoredSTH, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return loadLargestSTHFromDir(sthsDirPath)
+}
+
+// CountSTHs returns the number of STHs stored for the log.
+func (s *FilesystemState) CountSTHs(ctx context.Context, logID LogID) (int, error) {
+	return countFiles(filepath.Join(s.logStateDir(logID), "unverified_sths"), ".json")
+}
+
+// CountMalformedEntries returns the number of malformed entries recorded for the log.
+func (s *FilesystemState) CountMalformedEntries(ctx context.Context, logID LogID) (int, error) {
+	return countFiles(filepath.Join(s.logStateDir(logID), "malformed_entries"), ".json")
+}
+
+// CountHealthCheckFailures returns the number of health check failures recorded for the log.
+func (s *FilesystemState) CountHealthCheckFailures(ctx context.Context, logID LogID) (int, error) {
+	return countFiles(filepath.Join(s.logStateDir(logID), "healthchecks"), ".txt")
+}
+
+func (s *FilesystemState) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error {
+	filePath := filepath.Join(s.CacheDir, "issuers", hex.EncodeToString(fingerprint[:]))
+	return writeFile(filePath, issuer, 0666)
+}
+
+func (s *FilesystemState) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	filePath := filepath.Join(s.CacheDir, "issuers", hex.EncodeToString(fingerprint[:]))
+	issuer, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	} else {
+		return issuer, err
+	}
+}
+
+func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
+	var (
+		tbsFingerprint  = hex.EncodeToString(cert.TBSSHA256[:])
+		certFingerprint = hex.EncodeToString(cert.SHA256[:])
+	)
+	var (
+		tbsDir  = filepath.Join(s.StateDir, "certs", tbsFingerprint[0:2])
+		certDir = filepath.Join(s.StateDir, "certs", certFingerprint[0:2])
+	)
+
+	notifiedPath := filepath.Join(tbsDir, "."+tbsFingerprint+".notified")
+	if fileExists(notifiedPath) {
+		return nil
+	}
+	// previous versions of certspotter used these files to record that a notification was sent
+	for _, filename := range []string{
+		"." + certFingerprint + ".notified",
+		certFingerprint + ".cert.pem",
+		certFingerprint + ".precert.pem",
+	} {
+		if fileExists(filepath.Join(certDir, filename)) {
+			return nil
+		}
+	}
+
+	var paths *certPaths
+	if s.SaveCerts {
+		if err := os.Mkdir(tbsDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
+		}
+		if err := os.Mkdir(certDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
+		}
+		paths = &certPaths{
+			certPath: filepath.Join(certDir, certFingerprint+".pem"),
+			jsonPath: filepath.Join(certDir, certFingerprint+".v1.json"),
+			textPath: filepath.Join(certDir, certFingerprint+".txt"),
+		}
+		if err := writeCertFiles(cert, paths); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	} else {
+		paths = &certPaths{
+			temporary: true,
+
+			certPath: filepath.Join(os.TempDir(), "certspotter-"+rand.Text()+".pem"),
+			jsonPath: filepath.Join(os.TempDir(), "certspotter-"+rand.Text()+".v1.json"),
+			textPath: filepath.Join(os.TempDir(), "certspotter-"+rand.Text()+".txt"),
+		}
+		defer func() {
+			os.Remove(paths.certPath)
+			os.Remove(paths.jsonPath)
+			os.Remove(paths.textPath)
+		}()
+		if err := writeCertFiles(cert, paths); err != nil {
+			return fmt.Errorf("error saving certificate %x to temporary files: %w", cert.SHA256, err)
+		}
+	}
+
+	if err := s.notify(ctx, &notification{
+		summary: certNotificationSummary(cert),
+		environ: certNotificationEnviron(cert, paths),
+		text:    certNotificationText(cert, paths),
+	}); err != nil {
+		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
+	}
+
+	if s.SaveCerts {
+		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error {
+	var (
+		dirPath   = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
+		entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
+		textPath  = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
+	)
+
+	summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.GetMonitoringURL())
+	leafHash := merkletree.HashLeaf(entry.LeafInput())
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.GetMonitoringURL()))
+	writeField("Leaf Hash", leafHash.Base64String())
+	writeField("Error", parseError.Error())
+
+	if err := writeJSONFile(entryPath, entry.Entry, 0666); err != nil {
+		return fmt.Errorf("error saving JSON file: %w", err)
+	}
+	if err := writeTextFile(textPath, text.String(), 0666); err != nil {
+		return fmt.Errorf("error saving texT file: %w", err)
+	}
+
+	environ := []string{
+		"EVENT=malformed_cert",
+		"SUMMARY=" + summary,
+		"LOG_URI=" + entry.Log.GetMonitoringURL(),
+		"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
+		"LEAF_HASH=" + leafHash.Base64String(),
+		"PARSE_ERROR=" + parseError.Error(),
+		"ENTRY_FILENAME=" + entryPath,
+		"TEXT_FILENAME=" + textPath,
+		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
+	}
+
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: summary,
+		text:    text.String(),
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "healthchecks")
+	} else {
+		return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks")
+	}
+}
+
+func (s *FilesystemState) errorDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "errors")
+	}
+	return filepath.Join(s.logStateDir(ctlog.LogID), "errors")
+}
+
+func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
+	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
+	environ := []string{
+		"EVENT=error",
+		"SUMMARY=" + info.Summary(),
+		"TEXT_FILENAME=" + textPath,
+	}
+	text := info.Text()
+	if err := writeTextFile(textPath, text, 0666); err != nil {
+		return fmt.Errorf("error saving text file: %w", err)
+	}
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: info.Summary(),
+		text:    text,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error {
+	if ctlog == nil {
+		log.Print(notifyErr)
+	}
+
+	var (
+		now      = time.Now()
+		filePath = filepath.Join(s.errorDir(ctlog), now.Format(errorDateFormat))
+		line     = now.Format(time.RFC3339) + " " + notifyErr.Error() + "\n"
+	)
+
+	s.errorMu.Lock()
+	defer s.errorMu.Unlock()
+	file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if _, err := file.WriteString(line); err != nil {
+		return err
+	}
+	return file.Close()
+}
+
+func (s *FilesystemState) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
+	dir := s.errorDir(ctlog)
+	now := time.Now()
+	var buf []byte
+	for daysBack := 0; count > 0 && daysBack < keepErrorDays; daysBack++ {
+		datePath := filepath.Join(dir, now.AddDate(0, 0, -daysBack).Format(errorDateFormat))
+		dateBuf, dateLines, err := tailFile(datePath, count)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return "", err
+		}
+		buf = append(dateBuf, buf...)
+		count -= dateLines
+	}
+	return string(buf), nil
+}
+
+func (s *FilesystemState) PruneOldErrors() {
+	cutoff := time.Now().AddDate(0, 0, -keepErrorDays)
+	pruneDir := func(dir string) {
+		entries, err := os.ReadDir(dir)
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		} else if err != nil {
+			log.Printf("unable to read error directory: %s", err)
+			return
+		}
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			date, err := time.Parse(errorDateFormat, entry.Name())
+			if err != nil {
+				continue
+			}
+			if date.Before(cutoff) {
+				if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil && !errors.Is(err, fs.ErrNotExist) {
+					log.Printf("unable to remove old error file: %s", err)
+				}
+			}
+		}
+	}
+	pruneDir(filepath.Join(s.StateDir, "errors"))
+	logsDir := filepath.Join(s.StateDir, "logs")
+	logDirs, err := os.ReadDir(logsDir)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		log.Printf("unable to read logs directory: %s", err)
+		return
+	}
+	for _, d := range logDirs {
+		if !d.IsDir() {
+			continue
+		}
+		pruneDir(filepath.Join(logsDir, d.Name(), "errors"))
+	}
+}
@@ -0,0 +1,188 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+const recentErrorCount = 10
+
+func healthCheckFilename() string {
+	return time.Now().UTC().Format(time.RFC3339) + ".txt"
+}
+
+func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
+	var (
+		position    uint64
+		lastSuccess time.Time
+		verifiedSTH *cttypes.SignedTreeHead
+	)
+
+	if state, err := config.State.LoadLogState(ctx, ctlog.LogID); err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	} else if state != nil {
+		if time.Since(state.LastSuccess) < config.HealthCheckInterval {
+			// log is healthy
+			return nil
+		}
+
+		position = state.DownloadPosition.Size()
+		lastSuccess = state.LastSuccess
+		verifiedSTH = state.VerifiedSTH
+	}
+
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading STHs: %w", err)
+	}
+
+	var errorsDir string
+	if fsstate, ok := config.State.(*FilesystemState); ok {
+		errorsDir = fsstate.errorDir(ctlog)
+	}
+
+	if len(sths) == 0 {
+		errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
+		info := &StaleSTHInfo{
+			Log:          ctlog,
+			LastSuccess:  lastSuccess,
+			LatestSTH:    verifiedSTH,
+			RecentErrors: errors,
+			ErrorsDir:    errorsDir,
+		}
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
+			return fmt.Errorf("error notifying about stale STH: %w", err)
+		}
+	} else {
+		errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
+		info := &BacklogInfo{
+			Log:          ctlog,
+			LatestSTH:    sths[len(sths)-1],
+			Position:     position,
+			RecentErrors: errors,
+			ErrorsDir:    errorsDir,
+		}
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
+			return fmt.Errorf("error notifying about backlog: %w", err)
+		}
+	}
+
+	return nil
+}
+
+type HealthCheckFailure interface {
+	Summary() string
+	Text() string
+}
+
+type StaleSTHInfo struct {
+	Log          *loglist.Log
+	LastSuccess  time.Time               // may be zero
+	LatestSTH    *cttypes.SignedTreeHead // may be nil
+	RecentErrors string
+	ErrorsDir    string
+}
+
+type BacklogInfo struct {
+	Log          *loglist.Log
+	LatestSTH    *StoredSTH
+	Position     uint64
+	RecentErrors string
+	ErrorsDir    string
+}
+
+type StaleLogListInfo struct {
+	Source        string
+	LastSuccess   time.Time
+	RecentErrors  string
+	ErrorsDir     string
+}
+
+func (e *StaleSTHInfo) LastSuccessString() string {
+	if e.LastSuccess.IsZero() {
+		return "never"
+	} else {
+		return e.LastSuccess.String()
+	}
+}
+func (e *BacklogInfo) Backlog() uint64 {
+	return e.LatestSTH.TreeSize - e.Position
+}
+
+func (e *StaleSTHInfo) Summary() string {
+	return fmt.Sprintf("Unable to contact %s since %s", e.Log.GetMonitoringURL(), e.LastSuccessString())
+}
+func (e *BacklogInfo) Summary() string {
+	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.GetMonitoringURL())
+}
+func (e *StaleLogListInfo) Summary() string {
+	return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
+}
+
+func (e *StaleSTHInfo) Text() string {
+	text := new(strings.Builder)
+	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL(), e.LastSuccessString())
+	fmt.Fprintf(text, "\n")
+	if e.LatestSTH != nil {
+		fmt.Fprintf(text, "Latest known log size = %d\n", e.LatestSTH.TreeSize)
+	} else {
+		fmt.Fprintf(text, "Latest known log size = none\n")
+	}
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
+	return text.String()
+}
+func (e *BacklogInfo) Text() string {
+	text := new(strings.Builder)
+	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL())
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.StoredAt)
+	fmt.Fprintf(text, "Current position = %d\n", e.Position)
+	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
+	return text.String()
+}
+func (e *StaleLogListInfo) Text() string {
+	text := new(strings.Builder)
+	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
+	return text.String()
+}
+
+// TODO-3: make the errors more actionable
@@ -0,0 +1,45 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type LogID = cttypes.LogID
+
+func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) {
+	list, newToken, err := loglist.LoadIfModified(ctx, source, token)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	logs := make(map[LogID]*loglist.Log)
+	for operatorIndex := range list.Operators {
+		for logIndex := range list.Operators[operatorIndex].Logs {
+			log := &list.Operators[operatorIndex].Logs[logIndex]
+			if _, exists := logs[log.LogID]; exists {
+				return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
+			}
+			logs[log.LogID] = log
+		}
+		for logIndex := range list.Operators[operatorIndex].TiledLogs {
+			log := &list.Operators[operatorIndex].TiledLogs[logIndex]
+			if _, exists := logs[log.LogID]; exists {
+				return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
+			}
+			logs[log.LogID] = log
+		}
+	}
+	return logs, newToken, nil
+}
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+)
+
+const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700"
+
+func generateMessageID() string {
+	var randomBytes [16]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org"
+}
+
+func sendmailPath() string {
+	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
+		return envVar
+	} else {
+		return "/usr/sbin/sendmail"
+	}
+}
@@ -0,0 +1,608 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
+	"log"
+	mathrand "math/rand/v2"
+	"net/url"
+	"slices"
+	"time"
+
+	"software.sslmate.com/src/certspotter/ctclient"
+	"software.sslmate.com/src/certspotter/ctcrypto"
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+	"software.sslmate.com/src/certspotter/sequencer"
+)
+
+const (
+	getSTHInterval    = 5 * time.Minute
+	maxPartialTileAge = 5 * time.Minute
+)
+
+func downloadJobSize(ctlog *loglist.Log) uint64 {
+	if ctlog.IsStaticCTAPI() {
+		return ctclient.StaticTileWidth
+	} else if ctlog.CertspotterDownloadSize != 0 {
+		return uint64(ctlog.CertspotterDownloadSize)
+	} else {
+		return 1000
+	}
+}
+
+func downloadWorkers(ctlog *loglist.Log) int {
+	if ctlog.CertspotterDownloadQPS != 0 {
+		// parallelism is effectively governed by the rate limit so for now just hard code a number here
+		return 10
+	} else if ctlog.CertspotterDownloadJobs != 0 {
+		return ctlog.CertspotterDownloadJobs
+	} else {
+		return 1
+	}
+}
+
+func downloadRateLimit(ctlog *loglist.Log) rate.Limit {
+	if ctlog.CertspotterDownloadQPS != 0 {
+		return rate.Limit(ctlog.CertspotterDownloadQPS)
+	} else {
+		return rate.Inf
+	}
+}
+
+type verifyEntriesError struct {
+	sth             *cttypes.SignedTreeHead
+	entriesRootHash merkletree.Hash
+}
+
+func (e *verifyEntriesError) Error() string {
+	return fmt.Sprintf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", e.sth.TreeSize, e.sth.RootHash, e.entriesRootHash)
+}
+
+type partialError struct {
+	error
+}
+
+func (e partialError) Unwrap() error {
+	return e.error
+}
+
+func isPartialError(err error) bool {
+	var pe partialError
+	return errors.As(err, &pe)
+}
+
+func withRetry(ctx context.Context, config *Config, ctlog *loglist.Log, maxRetries int, f func() error) error {
+	minSleep := 100 * time.Millisecond
+	numRetries := 0
+	for ctx.Err() == nil {
+		err := f()
+		if err == nil || errors.Is(err, context.Canceled) {
+			return err
+		}
+		if numRetries > 0 {
+			err = fmt.Errorf("%w (retried %d times)", err, numRetries)
+		}
+		if isPartialError(err) {
+			recordError(ctx, config, ctlog, err)
+			return nil
+		}
+		if maxRetries != -1 && numRetries >= maxRetries {
+			return err
+		}
+		recordError(ctx, config, ctlog, err)
+		sleepTime := minSleep + mathrand.N(minSleep)
+		if err := sleep(ctx, sleepTime); err != nil {
+			return err
+		}
+		minSleep = min(minSleep*2, 30*time.Second)
+		numRetries++
+	}
+	return ctx.Err()
+}
+
+func getEntriesFull(ctx context.Context, client ctclient.Log, startInclusive, endExclusive uint64) ([]ctclient.Entry, error) {
+	allEntries := make([]ctclient.Entry, 0, endExclusive-startInclusive)
+	for startInclusive < endExclusive {
+		entries, err := client.GetEntries(ctx, startInclusive, endExclusive-1)
+		if err != nil {
+			return nil, err
+		}
+		allEntries = append(allEntries, entries...)
+		startInclusive += uint64(len(entries))
+	}
+	return allEntries, nil
+}
+
+func getAuthenticSTH(ctx context.Context, ctlog *loglist.Log, client ctclient.Log) (*cttypes.SignedTreeHead, string, error) {
+	sth, url, err := client.GetSTH(ctx)
+	if err != nil {
+		return nil, "", err
+	}
+	if err := ctcrypto.PublicKey(ctlog.Key).Verify(ctcrypto.SignatureInputForSTH(sth), sth.Signature); err != nil {
+		return nil, "", fmt.Errorf("STH has invalid signature: %w", err)
+	}
+	return sth, url, nil
+}
+
+type logClient struct {
+	config *Config
+	log    *loglist.Log
+	client ctclient.Log
+	lim    *rate.Limiter
+}
+
+func (client *logClient) GetSTH(ctx context.Context) (sth *cttypes.SignedTreeHead, url string, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
+		sth, url, err = getAuthenticSTH(ctx, client.log, client.client)
+		return err
+	})
+	return
+}
+func (client *logClient) GetRoots(ctx context.Context) (roots [][]byte, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
+		roots, err = client.client.GetRoots(ctx)
+		return err
+	})
+	return
+}
+func (client *logClient) GetEntries(ctx context.Context, startInclusive, endInclusive uint64) (entries []ctclient.Entry, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
+		entries, err = client.client.GetEntries(ctx, startInclusive, endInclusive)
+		if err != nil && len(entries) != 0 {
+			err = partialError{err}
+		}
+		return err
+	})
+	return
+}
+func (client *logClient) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (tree *merkletree.CollapsedTree, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
+		tree, err = client.client.ReconstructTree(ctx, sth)
+		return err
+	})
+	return
+}
+
+type issuerGetter struct {
+	config    *Config
+	log       *loglist.Log
+	logGetter ctclient.IssuerGetter
+}
+
+func (ig *issuerGetter) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	if issuer, err := ig.config.State.LoadIssuer(ctx, fingerprint); err != nil {
+		log.Printf("error loading cached issuer %x (issuer will be retrieved from log instead): %s", *fingerprint, err)
+	} else if issuer != nil {
+		return issuer, nil
+	}
+
+	var issuer []byte
+	if err := withRetry(ctx, ig.config, ig.log, 7, func() error {
+		var err error
+		issuer, err = ig.logGetter.GetIssuer(ctx, fingerprint)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	if err := ig.config.State.StoreIssuer(ctx, fingerprint, issuer); err != nil {
+		log.Printf("error caching issuer %x (issuer will be re-retrieved from log in the future): %s", *fingerprint, err)
+	}
+
+	return issuer, nil
+}
+
+func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.IssuerGetter, error) {
+	switch {
+	case ctlog.IsRFC6962():
+		logURL, err := url.Parse(ctlog.URL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("log has invalid URL: %w", err)
+		}
+		return &logClient{
+			config: config,
+			log:    ctlog,
+			client: &ctclient.RFC6962Log{URL: logURL},
+			lim:    rate.NewLimiter(downloadRateLimit(ctlog), 1),
+		}, nil, nil
+	case ctlog.IsStaticCTAPI():
+		submissionURL, err := url.Parse(ctlog.SubmissionURL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("log has invalid submission URL: %w", err)
+		}
+		monitoringURL, err := url.Parse(ctlog.MonitoringURL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("log has invalid monitoring URL: %w", err)
+		}
+		client := &ctclient.StaticLog{
+			SubmissionURL: submissionURL,
+			MonitoringURL: monitoringURL,
+			ID:            ctlog.LogID,
+		}
+		return &logClient{
+				config: config,
+				log:    ctlog,
+				client: client,
+				lim:    rate.NewLimiter(downloadRateLimit(ctlog), 1),
+			}, &issuerGetter{
+				config:    config,
+				log:       ctlog,
+				logGetter: client,
+			}, nil
+	default:
+		return nil, nil, errors.New("log uses unknown protocol")
+	}
+}
+
+func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) (returnedErr error) {
+	client, issuerGetter, err := newLogClient(config, ctlog)
+	if err != nil {
+		return err
+	}
+	if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
+	}
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	}
+	if state == nil {
+		if config.StartAtEnd {
+			sth, _, err := client.GetSTH(ctx)
+			if err != nil {
+				return err
+			}
+			tree, err := client.ReconstructTree(ctx, sth)
+			if err != nil {
+				return err
+			}
+			state = &LogState{
+				DownloadPosition: tree,
+				VerifiedPosition: tree,
+				VerifiedSTH:      sth,
+				LastSuccess:      time.Now(),
+			}
+		} else {
+			state = &LogState{
+				DownloadPosition: merkletree.EmptyCollapsedTree(),
+				VerifiedPosition: merkletree.EmptyCollapsedTree(),
+				VerifiedSTH:      nil,
+				LastSuccess:      time.Now(),
+			}
+		}
+		if config.Verbose {
+			log.Printf("%s: monitoring brand new log starting from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
+		}
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+	} else {
+		if config.Verbose {
+			log.Printf("%s: resuming monitoring from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
+		}
+	}
+
+	defer func() {
+		storeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		if err := config.State.StoreLogState(storeCtx, ctlog.LogID, state); err != nil && returnedErr == nil {
+			returnedErr = fmt.Errorf("error storing log state: %w", err)
+		}
+	}()
+
+retry:
+	position := state.DownloadPosition.Size()
+
+	// logs are monitored using the following pipeline of workers, with each worker sending results to the next worker:
+	//     1 getSTHWorker ==> 1 generateBatchesWorker ==> multiple downloadWorkers ==> multiple processWorkers ==> 1 saveStateWorker
+	// getSTHWorker          - periodically download STHs from the log
+	// generateBatchesWorker - generate batches of work
+	// downloadWorkers       - download the entries in each batch
+	// processWorkers        - process the certificates (store/notify if matches watch list) in each batch
+	// saveStateWorker       - builds the Merkle Tree and compares against STHs
+
+	sths := make(chan *cttypes.SignedTreeHead, 1)
+	batches := make(chan *batch, downloadWorkers(ctlog))
+	processedBatches := sequencer.New[batch](0, uint64(downloadWorkers(ctlog))*10)
+
+	group, gctx := errgroup.WithContext(ctx)
+	group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client, sths) })
+	group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, sths, batches) })
+	for range downloadWorkers(ctlog) {
+		downloadedBatches := make(chan *batch, 1)
+		group.Go(func() error { return downloadWorker(gctx, config, ctlog, client, batches, downloadedBatches) })
+		group.Go(func() error {
+			return processWorker(gctx, config, ctlog, issuerGetter, downloadedBatches, processedBatches)
+		})
+	}
+	group.Go(func() error { return saveStateWorker(gctx, config, ctlog, state, processedBatches) })
+
+	err = group.Wait()
+	if verifyErr := (*verifyEntriesError)(nil); errors.As(err, &verifyErr) {
+		recordError(ctx, config, ctlog, verifyErr)
+		state.rewindDownloadPosition()
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+		if err := sleep(ctx, 5*time.Minute); err != nil {
+			return err
+		}
+		goto retry
+	}
+	return err
+}
+
+func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, sthsOut chan<- *cttypes.SignedTreeHead) error {
+	ticker := time.NewTicker(getSTHInterval)
+	defer ticker.Stop()
+	for {
+		sth, _, err := client.GetSTH(ctx)
+		if err != nil {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case sthsOut <- sth:
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}
+
+type batch struct {
+	number       uint64
+	begin, end   uint64
+	discoveredAt time.Time        // time at which we became aware of the log having entries in range [begin,end)
+	sths         []*StoredSTH     // STHs with sizes in range [begin,end], sorted by TreeSize
+	entries      []ctclient.Entry // in range [begin,end)
+}
+
+// Create a batch starting from begin, based on sths (which must be non-empty, sorted by TreeSize, and contain only STHs with TreeSize >= begin).  Returns the batch, plus the remaining STHs.
+func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize uint64) (*batch, []*StoredSTH) {
+	batch := &batch{
+		number:       number,
+		begin:        begin,
+		discoveredAt: sths[0].StoredAt,
+	}
+	maxEnd := (begin/downloadJobSize + 1) * downloadJobSize
+	for _, sth := range sths {
+		if sth.StoredAt.Before(batch.discoveredAt) {
+			batch.discoveredAt = sth.StoredAt
+		}
+		if sth.TreeSize <= maxEnd {
+			batch.end = sth.TreeSize
+			batch.sths = append(batch.sths, sth)
+		} else {
+			batch.end = maxEnd
+			break
+		}
+	}
+	return batch, sths[len(batch.sths):]
+}
+
+// insert sth into sths, which is sorted by TreeSize, and return a new, still-sorted slice.
+// if an equivalent STH is already in sths, it is returned unchanged.
+func insertSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH {
+	i := len(sths)
+	for i > 0 {
+		if sths[i-1].Same(&sth.SignedTreeHead) {
+			return sths
+		}
+		if sths[i-1].TreeSize < sth.TreeSize {
+			break
+		}
+		i--
+	}
+	return slices.Insert(sths, i, sth)
+}
+
+func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, sthsIn <-chan *cttypes.SignedTreeHead, batchesOut chan<- *batch) error {
+	downloadJobSize := downloadJobSize(ctlog)
+
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading STHs: %w", err)
+	}
+	// sths is sorted by TreeSize but may contain STHs with TreeSize < position; get rid of them
+	for len(sths) > 0 && sths[0].TreeSize < position {
+		// TODO-4: audit sths[0] against log's verified STH
+		if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sths[0].SignedTreeHead); err != nil {
+			return fmt.Errorf("error removing STH: %w", err)
+		}
+		sths = sths[1:]
+	}
+	// from this point, sths is sorted by TreeSize and contains only STHs with TreeSize >= position
+	handleSTH := func(sth *cttypes.SignedTreeHead) error {
+		if sth.TreeSize < position {
+			// TODO-4: audit against log's verified STH
+		} else {
+			storedSTH, err := config.State.StoreSTH(ctx, ctlog.LogID, sth)
+			if err != nil {
+				return fmt.Errorf("error storing STH: %w", err)
+			}
+			sths = insertSTH(sths, storedSTH)
+		}
+		return nil
+	}
+
+	var number uint64
+	for {
+		for len(sths) == 0 {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case sth := <-sthsIn:
+				if err := handleSTH(sth); err != nil {
+					return err
+				}
+			}
+		}
+
+		batch, remainingSTHs := newBatch(number, position, sths, downloadJobSize)
+
+		if ctlog.IsStaticCTAPI() && batch.end%downloadJobSize != 0 {
+			// Wait to download this partial tile until it's old enough
+			if age := time.Since(batch.discoveredAt); age < maxPartialTileAge {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-time.After(maxPartialTileAge - age):
+				case sth := <-sthsIn:
+					if err := handleSTH(sth); err != nil {
+						return err
+					}
+					continue
+				}
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case sth := <-sthsIn:
+			if err := handleSTH(sth); err != nil {
+				return err
+			}
+		case batchesOut <- batch:
+			number = batch.number + 1
+			position = batch.end
+			sths = remainingSTHs
+		}
+	}
+}
+
+func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, batchesIn <-chan *batch, batchesOut chan<- *batch) error {
+	for {
+		var batch *batch
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case batch = <-batchesIn:
+		}
+
+		entries, err := getEntriesFull(ctx, client, batch.begin, batch.end)
+		if err != nil {
+			return err
+		}
+		batch.entries = entries
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case batchesOut <- batch:
+		}
+	}
+}
+
+func processWorker(ctx context.Context, config *Config, ctlog *loglist.Log, issuerGetter ctclient.IssuerGetter, batchesIn <-chan *batch, batchesOut *sequencer.Channel[batch]) error {
+	for {
+		var batch *batch
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case batch = <-batchesIn:
+		}
+		for offset, entry := range batch.entries {
+			index := batch.begin + uint64(offset)
+			if err := processLogEntry(ctx, config, issuerGetter, &LogEntry{
+				Entry: entry,
+				Index: index,
+				Log:   ctlog,
+			}); err != nil {
+				return fmt.Errorf("error processing entry %d: %w", index, err)
+			}
+		}
+		if err := batchesOut.Add(ctx, batch.number, batch); err != nil {
+			return err
+		}
+	}
+}
+
+func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, state *LogState, batchesIn *sequencer.Channel[batch]) error {
+	for {
+		batch, err := batchesIn.Next(ctx)
+		if err != nil {
+			return err
+		}
+		if batch.begin != state.DownloadPosition.Size() {
+			panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin))
+		}
+		for {
+			for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() {
+				sth := batch.sths[0]
+				batch.sths = batch.sths[1:]
+				if rootHash := state.DownloadPosition.CalculateRoot(); sth.RootHash != rootHash {
+					return &verifyEntriesError{
+						sth:             &sth.SignedTreeHead,
+						entriesRootHash: rootHash,
+					}
+				}
+				state.advanceVerifiedPosition()
+				state.LastSuccess = sth.StoredAt
+				state.VerifiedSTH = &sth.SignedTreeHead
+				if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+					return fmt.Errorf("error storing log state: %w", err)
+				}
+				// don't remove the STH until state has been durably stored
+				if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sth.SignedTreeHead); err != nil {
+					return fmt.Errorf("error removing verified STH: %w", err)
+				}
+				if config.Verbose {
+					log.Printf("%s: verified position is now %d", ctlog.GetMonitoringURL(), sth.SignedTreeHead.TreeSize)
+				}
+			}
+			if len(batch.entries) == 0 {
+				break
+			}
+			entry := batch.entries[0]
+			batch.entries = batch.entries[1:]
+			leafHash := merkletree.HashLeaf(entry.LeafInput())
+			state.DownloadPosition.Add(leafHash)
+		}
+
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+	}
+}
+
+func sleep(ctx context.Context, duration time.Duration) error {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
@@ -0,0 +1,166 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+var stdoutMu sync.Mutex
+
+type notification struct {
+	environ []string
+	summary string
+	text    string
+}
+
+func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
+	if s.Stdout {
+		writeToStdout(notif)
+	}
+
+	if len(s.Email) > 0 {
+		if err := sendEmail(ctx, s.Email, notif); err != nil {
+			return err
+		}
+	}
+
+	if s.Script != "" {
+		if err := execScript(ctx, s.Script, notif); err != nil {
+			return err
+		}
+	}
+
+	if s.ScriptDir != "" {
+		if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writeToStdout(notif *notification) {
+	stdoutMu.Lock()
+	defer stdoutMu.Unlock()
+	os.Stdout.WriteString(notif.text + "\n")
+}
+
+func sendEmail(ctx context.Context, to []string, notif *notification) error {
+	stdin := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+
+	from := os.Getenv("EMAIL")
+
+	if from != "" {
+		fmt.Fprintf(stdin, "From: %s\n", from)
+	}
+	fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
+	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
+	fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
+	fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
+	fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
+	fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
+	fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
+	fmt.Fprintf(stdin, "\n")
+	fmt.Fprint(stdin, notif.text)
+
+	args := []string{"-i"}
+	if from != "" {
+		args = append(args, "-f", from)
+	}
+	args = append(args, "--")
+	args = append(args, to...)
+
+	sendmailCtx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Minute))
+	defer cancel()
+	sendmail := exec.CommandContext(sendmailCtx, sendmailPath(), args...)
+	sendmail.Stdin = stdin
+	sendmail.Stderr = stderr
+	sendmail.WaitDelay = 5 * time.Second
+
+	if err := sendmail.Run(); err == nil || err == exec.ErrWaitDelay {
+		return nil
+	} else if sendmailCtx.Err() != nil && ctx.Err() == nil {
+		return fmt.Errorf("error sending email to %v: sendmail command timed out", to)
+	} else if ctx.Err() != nil {
+		// if the context was canceled, we can't be sure that the error is the fault of sendmail, so ignore it
+		return ctx.Err()
+	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
+		return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
+	} else if isExitError {
+		return fmt.Errorf("error sending email to %v: sendmail terminated by signal with error %q", to, strings.TrimSpace(stderr.String()))
+	} else {
+		return fmt.Errorf("error sending email to %v: error running sendmail command: %w", to, err)
+	}
+}
+
+func execScript(ctx context.Context, scriptName string, notif *notification) error {
+	stderr := new(bytes.Buffer)
+
+	cmd := exec.CommandContext(ctx, scriptName)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, notif.environ...)
+	cmd.Stderr = stderr
+	cmd.WaitDelay = 5 * time.Second
+
+	if err := cmd.Run(); err == nil || err == exec.ErrWaitDelay {
+		return nil
+	} else if ctx.Err() != nil {
+		// if the context was canceled, we can't be sure that the error is the fault of the script, so ignore it
+		return ctx.Err()
+	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
+		return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
+	} else if isExitError {
+		return fmt.Errorf("script %q terminated by signal with error %q", scriptName, strings.TrimSpace(stderr.String()))
+	} else {
+		return fmt.Errorf("error executing script: %w", err)
+	}
+}
+
+func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
+	dirents, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err)
+	}
+	for _, dirent := range dirents {
+		if strings.HasPrefix(dirent.Name(), ".") {
+			continue
+		}
+		scriptPath := filepath.Join(dirPath, dirent.Name())
+		info, err := os.Stat(scriptPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err)
+		} else if info.Mode().IsRegular() && isExecutable(info.Mode()) {
+			if err := execScript(ctx, scriptPath, notif); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func isExecutable(mode os.FileMode) bool {
+	return mode&0111 != 0
+}
@@ -0,0 +1,151 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/ctclient"
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type LogEntry struct {
+	ctclient.Entry
+	Index uint64
+	Log   *loglist.Log
+}
+
+func processLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry) error {
+	leaf, err := cttypes.ParseLeafInput(entry.LeafInput())
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
+	}
+	switch leaf.TimestampedEntry.EntryType {
+	case cttypes.X509EntryType:
+		return processX509LogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryASN1Cert)
+	case cttypes.PrecertEntryType:
+		return processPrecertLogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryPreCert)
+	default:
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
+	}
+}
+
+func processX509LogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, cert *cttypes.ASN1Cert) error {
+	certInfo, err := certspotter.MakeCertInfoFromRawCert(*cert)
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
+	}
+	if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil {
+		certInfo.TBS = precertTBS
+	} else {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err))
+	}
+
+	getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) {
+		var (
+			chain = []cttypes.ASN1Cert{*cert}
+			errs  = []error{}
+		)
+		if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil {
+			chain = append(chain, issuers...)
+		} else {
+			errs = append(errs, err)
+		}
+		return chain, errors.Join(errs...)
+	}
+	return processCertificate(ctx, config, entry, certInfo, getChain)
+}
+
+func processPrecertLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, precert *cttypes.PreCert) error {
+	certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
+	}
+	precertBytes, err := entry.Precertificate()
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error getting precert entry's precertificate: %w", err))
+	}
+
+	getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) {
+		var (
+			chain = []cttypes.ASN1Cert{precertBytes}
+			errs  = []error{}
+		)
+		if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil {
+			chain = append(chain, issuers...)
+		} else {
+			errs = append(errs, err)
+		}
+		if _, err := certspotter.ValidatePrecert(precertBytes, precert.TBSCertificate); err != nil {
+			errs = append(errs, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
+		}
+		return chain, errors.Join(errs...)
+	}
+	return processCertificate(ctx, config, entry, certInfo, getChain)
+}
+
+func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, getChain func(context.Context) ([]cttypes.ASN1Cert, error)) error {
+	identifiers, err := certInfo.ParseIdentifiers()
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, err)
+	}
+	// CTCLOUD PATCH: hand identifiers directly to the in-process sink and skip
+	// the watchlist/chain/notification path.  See CTCLOUD_PATCH.md.
+	if config.IdentifierSink != nil {
+		config.IdentifierSink.HandleIdentifiers(ctx, identifiers)
+		return nil
+	}
+	matched, watchItem := config.WatchList.Matches(identifiers)
+	if !matched {
+		return nil
+	}
+
+	chain, chainErr := getChain(ctx)
+	if chainErr != nil {
+		if ctx.Err() != nil {
+			// Getting chain failed, but it was probably because our context
+			// has been canceled, so just act like we never called getChain.
+			return ctx.Err()
+		}
+		// Although getting the chain failed, we still want to notify
+		// the user about the matching certificate. We'll include chainErr in the
+		// notification so the user knows why the chain is missing or incorrect.
+	}
+
+	cert := &DiscoveredCert{
+		WatchItem:    watchItem,
+		LogEntry:     entry,
+		Info:         certInfo,
+		Chain:        chain,
+		ChainError:   chainErr,
+		TBSSHA256:    sha256.Sum256(certInfo.TBS.Raw),
+		SHA256:       sha256.Sum256(chain[0]),
+		PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes),
+		Identifiers:  identifiers,
+	}
+
+	if err := config.State.NotifyCert(ctx, cert); err != nil {
+		return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
+	}
+
+	return nil
+}
+
+func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
+	if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
+		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.GetMonitoringURL(), parseError, err)
+	}
+	return nil
+}
@@ -0,0 +1,91 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type LogState struct {
+	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
+	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
+	VerifiedSTH      *cttypes.SignedTreeHead   `json:"verified_sth"`
+	LastSuccess      time.Time                 `json:"last_success"`
+}
+
+func (state *LogState) rewindDownloadPosition() {
+	position := state.VerifiedPosition.Clone()
+	state.DownloadPosition = &position
+}
+
+func (state *LogState) advanceVerifiedPosition() {
+	position := state.DownloadPosition.Clone()
+	state.VerifiedPosition = &position
+}
+
+// Methods are safe to call concurrently.
+type StateProvider interface {
+	// Initialize the state.  Called before any other method in this interface.
+	// Idempotent: returns nil if the state is already initialized.
+	Prepare(context.Context) error
+
+	// Initialize the state for the given log.  Called before any other method
+	// with the log ID.  Idempotent: returns nil if log state already initialized.
+	PrepareLog(context.Context, LogID) error
+
+	// Store log state for retrieval by LoadLogState.
+	StoreLogState(context.Context, LogID, *LogState) error
+
+	// Load log state that was previously stored with StoreLogState.
+	// Returns nil, nil if StoreLogState has not been called yet for this log.
+	LoadLogState(context.Context, LogID) (*LogState, error)
+
+	// Store STH for retrieval by LoadSTHs.  If an STH with the same
+	// timestamp and root hash is already stored, this STH can be ignored.
+	StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) (*StoredSTH, error)
+
+	// Load all STHs for this log previously stored with StoreSTH.
+	// The returned slice must be sorted by tree size.
+	LoadSTHs(context.Context, LogID) ([]*StoredSTH, error)
+
+	// Remove an STH so it is no longer returned by LoadSTHs.
+	RemoveSTH(context.Context, LogID, *cttypes.SignedTreeHead) error
+
+	// Store a DER-encoded issuer certificate with the given fingerprint for
+	// retrieval by LoadIssuer.  Returns nil if the issuer has already been stored.
+	StoreIssuer(context.Context, *[32]byte, []byte) error
+
+	// Retrieve a DER-encoded issuer certificate previously stored with StoreIssuer.
+	// Returns nil, nil if this issuer certificate has not been stored.
+	LoadIssuer(context.Context, *[32]byte) ([]byte, error)
+
+	// Called when a certificate matching the watch list is discovered.
+	NotifyCert(context.Context, *DiscoveredCert) error
+
+	// Called when certspotter fails to parse a log entry.
+	NotifyMalformedEntry(context.Context, *LogEntry, error) error
+
+	// Called when a health check fails.  The log is nil if the
+	// feailure is not associated with a log.
+	NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error
+
+	// Called when a non-fatal error occurs.  The log is nil if the error is
+	// not associated with a log.  Note that most errors are transient, and
+	// certspotter will retry the failed operation later.
+	NotifyError(context.Context, *loglist.Log, error) error
+
+	// Retrieve the specified number of most recent errors.
+	GetErrors(context.Context, *loglist.Log, int) (string, error)
+}
@@ -0,0 +1,167 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func readVersion(stateDir string) (int, error) {
+	path := filepath.Join(stateDir, "version")
+
+	fileBytes, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		if fileExists(filepath.Join(stateDir, "evidence")) {
+			return 0, nil
+		} else {
+			return -1, nil
+		}
+	} else if err != nil {
+		return -1, err
+	}
+
+	version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes)))
+	if err != nil {
+		return -1, fmt.Errorf("version file %q is malformed: %w", path, err)
+	}
+
+	return version, nil
+}
+
+func writeVersion(stateDir string) error {
+	return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666)
+}
+
+func migrateLogStateDirV1(dir string) error {
+	var sth cttypes.SignedTreeHead
+	var tree merkletree.CollapsedTree
+
+	sthPath := filepath.Join(dir, "sth.json")
+	sthData, err := os.ReadFile(sthPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	treePath := filepath.Join(dir, "tree.json")
+	treeData, err := os.ReadFile(treePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(sthData, &sth); err != nil {
+		return fmt.Errorf("error unmarshaling %s: %w", sthPath, err)
+	}
+	if err := json.Unmarshal(treeData, &tree); err != nil {
+		return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
+	}
+
+	stateFile := LogState{
+		DownloadPosition: &tree,
+		VerifiedPosition: &tree,
+		VerifiedSTH:      &sth,
+		LastSuccess:      time.Now(),
+	}
+	if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
+		return err
+	}
+
+	if err := os.Remove(sthPath); err != nil {
+		return err
+	}
+	if err := os.Remove(treePath); err != nil {
+		return err
+	}
+	return nil
+}
+
+func migrateStateDirV1(stateDir string) error {
+	if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) {
+		return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile)
+	}
+
+	if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil {
+		for _, logDir := range logDirs {
+			if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() {
+				continue
+			}
+			if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil {
+				return fmt.Errorf("error migrating log state: %w", err)
+			}
+		}
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	if err := writeVersion(stateDir); err != nil {
+		return err
+	}
+
+	if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	return nil
+}
+
+func prepareStateDir(stateDir string) error {
+	if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+		return err
+	}
+
+	if version, err := readVersion(stateDir); err != nil {
+		return err
+	} else if version == -1 {
+		if err := writeVersion(stateDir); err != nil {
+			return err
+		}
+	} else if version == 0 {
+		return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir)
+	} else if version == 1 {
+		if err := migrateStateDirV1(stateDir); err != nil {
+			return err
+		}
+	} else if version > 2 {
+		return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
+	}
+
+	for _, subdir := range []string{"certs", "logs", "healthchecks", "errors"} {
+		if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func prepareCacheDir(cacheDir string) error {
+	if err := os.MkdirAll(cacheDir, 0777); err != nil {
+		return err
+	}
+	for _, subdir := range []string{"issuers"} {
+		if err := os.Mkdir(filepath.Join(cacheDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
@@ -0,0 +1,178 @@
+// Copyright (C) 2017, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"cmp"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"software.sslmate.com/src/certspotter/cttypes"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type StoredSTH struct {
+	cttypes.SignedTreeHead
+	StoredAt time.Time `json:"stored_at"` // time at which the STH was first stored
+}
+
+func loadSTHsFromDir(dirPath string) ([]*StoredSTH, error) {
+	entries, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return []*StoredSTH{}, nil
+	} else if err != nil {
+		return nil, err
+	}
+	sths := make([]*StoredSTH, 0, len(entries))
+	for _, entry := range entries {
+		filename := entry.Name()
+		if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
+			continue
+		}
+		sth, err := readSTHFile(filepath.Join(dirPath, filename))
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+		sths = append(sths, sth)
+	}
+	slices.SortFunc(sths, func(a, b *StoredSTH) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
+	return sths, nil
+}
+
+// loadLargestSTHFromDir returns the STH in dirPath with the largest tree size, breaking
+// ties by later timestamp.
+func loadLargestSTHFromDir(dirPath string) (*StoredSTH, error) {
+	compareSTH := func(a, b *StoredSTH) int {
+		return cmp.Or(
+			cmp.Compare(a.TreeSize, b.TreeSize),
+			cmp.Compare(a.Timestamp, b.Timestamp),
+		)
+	}
+
+	entries, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var largestSTH *StoredSTH
+	for _, entry := range entries {
+		filename := entry.Name()
+		if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
+			continue
+		}
+		if largestSTH != nil {
+			if treeSize, ok := sthFilenameTreeSize(filename); ok && treeSize < largestSTH.TreeSize {
+				continue
+			}
+		}
+		sth, err := readSTHFile(filepath.Join(dirPath, filename))
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+		if largestSTH == nil || compareSTH(sth, largestSTH) > 0 {
+			largestSTH = sth
+		}
+	}
+	return largestSTH, nil
+}
+
+func readSTHFile(filePath string) (*StoredSTH, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+	sth := &StoredSTH{
+		StoredAt: info.ModTime(),
+	}
+
+	fileBytes, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", filePath, err)
+	}
+	if err := json.Unmarshal(fileBytes, &sth.SignedTreeHead); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return sth, nil
+}
+
+func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) (*StoredSTH, error) {
+	filePath := filepath.Join(dirPath, sthFilename(sth))
+
+	if info, err := os.Lstat(filePath); err == nil {
+		return &StoredSTH{
+			SignedTreeHead: *sth,
+			StoredAt:       info.ModTime(),
+		}, nil
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return nil, err
+	}
+
+	if err := writeJSONFile(filePath, sth, 0666); err != nil {
+		return nil, err
+	}
+
+	return &StoredSTH{
+		SignedTreeHead: *sth,
+		StoredAt:       time.Now(), // not the exact modtime of the file, but close enough for our purposes
+	}, nil
+}
+
+func removeSTHFromDir(dirPath string, sth *cttypes.SignedTreeHead) error {
+	filePath := filepath.Join(dirPath, sthFilename(sth))
+	err := os.Remove(filePath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+	return nil
+}
+
+// generate a filename that uniquely identifies the STH (within the context of a particular log)
+func sthFilename(sth *cttypes.SignedTreeHead) string {
+	hasher := sha256.New()
+	binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
+	hasher.Write(sth.RootHash[:])
+	return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
+}
+
+// sthFilenameTreeSize returns the tree size encoded in the filename, or (0, false)
+// if the filename doesn't parse.
+func sthFilenameTreeSize(filename string) (uint64, bool) {
+	separator := strings.IndexByte(filename, '-')
+	if separator <= 0 {
+		return 0, false
+	}
+	treeSize, err := strconv.ParseUint(filename[:separator], 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	return treeSize, true
+}
@@ -0,0 +1,135 @@
+// Copyright (C) 2016, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bufio"
+	"fmt"
+	"golang.org/x/net/idna"
+	"io"
+	"software.sslmate.com/src/certspotter"
+	"strings"
+)
+
+type WatchItem struct {
+	domain       []string
+	acceptSuffix bool
+}
+
+type WatchList []WatchItem
+
+func ParseWatchItem(str string) (WatchItem, error) {
+	fields := strings.Fields(str)
+	if len(fields) == 0 {
+		return WatchItem{}, fmt.Errorf("empty domain")
+	}
+	domain := fields[0]
+
+	for _, field := range fields[1:] {
+		switch {
+		case strings.HasPrefix(field, "valid_at:"):
+			// Ignore for backwards compatibility
+		default:
+			return WatchItem{}, fmt.Errorf("unknown parameter %q", field)
+		}
+	}
+
+	if domain == "." {
+		// "." as in root zone -> matches everything
+		return WatchItem{
+			domain:       []string{},
+			acceptSuffix: true,
+		}, nil
+	}
+
+	acceptSuffix := false
+	if strings.HasPrefix(domain, ".") {
+		acceptSuffix = true
+		domain = domain[1:]
+	}
+
+	asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, ".")))
+	if err != nil {
+		return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err)
+	}
+	return WatchItem{
+		domain:       strings.Split(asciiDomain, "."),
+		acceptSuffix: acceptSuffix,
+	}, nil
+}
+
+func ReadWatchList(reader io.Reader) (WatchList, error) {
+	items := make(WatchList, 0, 50)
+	scanner := bufio.NewScanner(reader)
+	lineNo := 0
+	for scanner.Scan() {
+		line := scanner.Text()
+		lineNo++
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		item, err := ParseWatchItem(line)
+		if err != nil {
+			return nil, fmt.Errorf("%w on line %d", err, lineNo)
+		}
+		items = append(items, item)
+	}
+	return items, scanner.Err()
+}
+
+func (item WatchItem) String() string {
+	if item.acceptSuffix {
+		return "." + strings.Join(item.domain, ".")
+	} else {
+		return strings.Join(item.domain, ".")
+	}
+}
+
+func (item WatchItem) matchesDNSName(dnsName []string) bool {
+	watchDomain := item.domain
+	for len(dnsName) > 0 && len(watchDomain) > 0 {
+		certLabel := dnsName[len(dnsName)-1]
+		watchLabel := watchDomain[len(watchDomain)-1]
+
+		if !dnsLabelMatches(certLabel, watchLabel) {
+			return false
+		}
+
+		dnsName = dnsName[:len(dnsName)-1]
+		watchDomain = watchDomain[:len(watchDomain)-1]
+	}
+	return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0)
+}
+
+func dnsLabelMatches(certLabel string, watchLabel string) bool {
+	// For fail-safe behavior, if a label was unparsable, it matches everything.
+	// Similarly, redacted labels match everything, since the label _might_ be
+	// for a name we're interested in.
+
+	return certLabel == "*" ||
+		certLabel == "?" ||
+		certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
+		certspotter.MatchesWildcard(watchLabel, certLabel)
+}
+
+func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) {
+	dnsNames := make([][]string, len(identifiers.DNSNames))
+	for i, dnsName := range identifiers.DNSNames {
+		dnsNames[i] = strings.Split(dnsName, ".")
+	}
+	for _, item := range list {
+		for _, dnsName := range dnsNames {
+			if item.matchesDNSName(dnsName) {
+				return true, item
+			}
+		}
+	}
+	return false, WatchItem{}
+}