StasonJatham
e8f7d6bfdf
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
92 lines
3.4 KiB
Go
92 lines
3.4 KiB
Go
// Package metrics exposes lightweight Prometheus text-format metrics without
|
|
// pulling in a metrics dependency. Counters are plain atomics; gauges are
|
|
// callback functions so they always report current values.
|
|
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"runtime"
|
|
"sync/atomic"
|
|
)
|
|
|
|
type gauge struct {
|
|
name string
|
|
help string
|
|
extra string
|
|
fn func() float64
|
|
}
|
|
|
|
type Metrics struct {
|
|
CTCertificates atomic.Uint64
|
|
CTIdentifiers atomic.Uint64
|
|
|
|
DirectMatches atomic.Uint64
|
|
CNAMEMatches atomic.Uint64
|
|
|
|
DNSQueries atomic.Uint64
|
|
DNSCacheHits atomic.Uint64
|
|
DNSErrors atomic.Uint64
|
|
|
|
Candidates atomic.Uint64
|
|
|
|
DBRows atomic.Uint64
|
|
DBErrors atomic.Uint64
|
|
|
|
QueueDrops atomic.Uint64
|
|
QueueFull atomic.Uint64
|
|
MalformedCT atomic.Uint64
|
|
CTErrors atomic.Uint64
|
|
ProbeAttempts atomic.Uint64
|
|
ProbePublic atomic.Uint64
|
|
|
|
gauges []gauge
|
|
}
|
|
|
|
func New() *Metrics {
|
|
m := &Metrics{}
|
|
m.gauges = []gauge{
|
|
{"go_goroutines", "Number of goroutines", "", func() float64 { return float64(runtime.NumGoroutine()) }},
|
|
{"process_system_bytes", "Bytes of memory obtained from the OS", "", func() float64 {
|
|
var ms runtime.MemStats
|
|
runtime.ReadMemStats(&ms)
|
|
return float64(ms.Sys)
|
|
}},
|
|
}
|
|
return m
|
|
}
|
|
|
|
// SetQueueDepth registers a gauge reporting the live depth of a named queue.
|
|
func (m *Metrics) SetQueueDepth(name string, fn func() float64) {
|
|
m.gauges = append(m.gauges, gauge{"queue_depth", "Current depth of an internal queue", `{queue="` + name + `"}`, fn})
|
|
}
|
|
|
|
// Handler returns an http.Handler serving the Prometheus text exposition format.
|
|
func (m *Metrics) Handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
|
counter := func(name, help string, v uint64) {
|
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v)
|
|
}
|
|
counter("ct_certificates_total", "Certificates processed from CT logs", m.CTCertificates.Load())
|
|
counter("ct_identifiers_total", "DNS identifiers parsed from CT certificates", m.CTIdentifiers.Load())
|
|
counter("classifier_direct_matches_total", "Identifiers matching a provider directly", m.DirectMatches.Load())
|
|
counter("classifier_cname_matches_total", "CNAME targets matching a provider", m.CNAMEMatches.Load())
|
|
counter("dns_queries_total", "DNS queries performed", m.DNSQueries.Load())
|
|
counter("dns_cache_hits_total", "DNS lookups served from cache", m.DNSCacheHits.Load())
|
|
counter("dns_errors_total", "DNS lookup errors", m.DNSErrors.Load())
|
|
counter("candidates_total", "Provider candidates produced", m.Candidates.Load())
|
|
counter("db_rows_total", "Rows written to PostgreSQL", m.DBRows.Load())
|
|
counter("db_errors_total", "PostgreSQL write errors", m.DBErrors.Load())
|
|
counter("queue_drops_total", "Items dropped because a queue was full", m.QueueDrops.Load())
|
|
counter("queue_full_total", "Times a queue was observed full", m.QueueFull.Load())
|
|
counter("ct_malformed_entries_total", "Malformed CT log entries", m.MalformedCT.Load())
|
|
counter("ct_errors_total", "Non-fatal CT errors", m.CTErrors.Load())
|
|
counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
|
|
counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
|
|
for _, g := range m.gauges {
|
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
|
|
}
|
|
})
|
|
}
|