StasonJatham
a366905a43
Logging: - structured slog with LOG_FORMAT=json|text and a stable key vocabulary - periodic stats record with queue depths, db pool, goroutines, memory, dns/probe - no per-certificate logging (hot path is counters only) UI (Bucketeer branding, DESIGN.md): - light+dark themes (OS default, persisted header toggle, no first-paint flash) - official mascot: lossless WebP light/dark logos + favicon, embedded and served with cache headers (1.2MB source PNGs -> ~27/22KB webp) - Assets/Contents views, pagination (offset) and native virtual scrolling (content-visibility) so large result sets do not blow up the DOM - search with debounce/abort + URL state, exposure chips, sort, summary counts Readability + indexing: - internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch (metadata only: keys/sizes/dates; never object contents) - asset_entries table + store write/search; readable resources are indexed and searchable in the UI - deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING, PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune HTTP reads - Azure account-level listing fixed: 403 is reported reachable, not auth_required, since anonymous access is per-container DNS: - curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at once broke the container resolver) - random healthy selection + failover + per-server cooldown Other: prober Close fixes a shutdown hang; config/README updated.
94 lines
3.5 KiB
Go
94 lines
3.5 KiB
Go
// Package metrics exposes lightweight Prometheus text-format metrics without
|
|
// pulling in a metrics dependency. Counters are plain atomics; gauges are
|
|
// callback functions so they always report current values.
|
|
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"runtime"
|
|
"sync/atomic"
|
|
)
|
|
|
|
type gauge struct {
|
|
name string
|
|
help string
|
|
extra string
|
|
fn func() float64
|
|
}
|
|
|
|
type Metrics struct {
|
|
CTCertificates atomic.Uint64
|
|
CTIdentifiers atomic.Uint64
|
|
|
|
DirectMatches atomic.Uint64
|
|
CNAMEMatches atomic.Uint64
|
|
|
|
DNSQueries atomic.Uint64
|
|
DNSCacheHits atomic.Uint64
|
|
DNSErrors atomic.Uint64
|
|
|
|
Candidates atomic.Uint64
|
|
|
|
DBRows atomic.Uint64
|
|
DBErrors atomic.Uint64
|
|
|
|
QueueDrops atomic.Uint64
|
|
QueueFull atomic.Uint64
|
|
MalformedCT atomic.Uint64
|
|
CTErrors atomic.Uint64
|
|
ProbeAttempts atomic.Uint64
|
|
ProbePublic atomic.Uint64
|
|
EntriesIndexed atomic.Uint64
|
|
|
|
gauges []gauge
|
|
}
|
|
|
|
func New() *Metrics {
|
|
m := &Metrics{}
|
|
m.gauges = []gauge{
|
|
{"go_goroutines", "Number of goroutines", "", func() float64 { return float64(runtime.NumGoroutine()) }},
|
|
{"process_system_bytes", "Bytes of memory obtained from the OS", "", func() float64 {
|
|
var ms runtime.MemStats
|
|
runtime.ReadMemStats(&ms)
|
|
return float64(ms.Sys)
|
|
}},
|
|
}
|
|
return m
|
|
}
|
|
|
|
// SetQueueDepth registers a gauge reporting the live depth of a named queue.
|
|
func (m *Metrics) SetQueueDepth(name string, fn func() float64) {
|
|
m.gauges = append(m.gauges, gauge{"queue_depth", "Current depth of an internal queue", `{queue="` + name + `"}`, fn})
|
|
}
|
|
|
|
// Handler returns an http.Handler serving the Prometheus text exposition format.
|
|
func (m *Metrics) Handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
|
counter := func(name, help string, v uint64) {
|
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v)
|
|
}
|
|
counter("ct_certificates_total", "Certificates processed from CT logs", m.CTCertificates.Load())
|
|
counter("ct_identifiers_total", "DNS identifiers parsed from CT certificates", m.CTIdentifiers.Load())
|
|
counter("classifier_direct_matches_total", "Identifiers matching a provider directly", m.DirectMatches.Load())
|
|
counter("classifier_cname_matches_total", "CNAME targets matching a provider", m.CNAMEMatches.Load())
|
|
counter("dns_queries_total", "DNS queries performed", m.DNSQueries.Load())
|
|
counter("dns_cache_hits_total", "DNS lookups served from cache", m.DNSCacheHits.Load())
|
|
counter("dns_errors_total", "DNS lookup errors", m.DNSErrors.Load())
|
|
counter("candidates_total", "Provider candidates produced", m.Candidates.Load())
|
|
counter("db_rows_total", "Rows written to PostgreSQL", m.DBRows.Load())
|
|
counter("db_errors_total", "PostgreSQL write errors", m.DBErrors.Load())
|
|
counter("queue_drops_total", "Items dropped because a queue was full", m.QueueDrops.Load())
|
|
counter("queue_full_total", "Times a queue was observed full", m.QueueFull.Load())
|
|
counter("ct_malformed_entries_total", "Malformed CT log entries", m.MalformedCT.Load())
|
|
counter("ct_errors_total", "Non-fatal CT errors", m.CTErrors.Load())
|
|
counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
|
|
counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
|
|
counter("entries_indexed_total", "Metadata entries indexed from readable resources", m.EntriesIndexed.Load())
|
|
for _, g := range m.gauges {
|
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
|
|
}
|
|
})
|
|
}
|