StasonJatham
a899362e18
CMS SANs are custom domains, not bucket names, so matching SANs finds almost no buckets. This adds internal/discover: from each non-matching hostname it derives candidate names (domain label + subdomain label + ~40 suffixes) plus a built-in wordlist, and anonymously tests listing on S3, GCS and (optionally) DigitalOcean Spaces. Every 200 is indexed (keys/sizes/dates only). Bounded hard: global BUCKET_SCAN_QPS rate limit, 12h per-name dedup, random sampled domain queue, MaxPerDomain cap. Never downloads contents. Fix: GCS candidates now use the virtual-host form (<bucket>.storage.googleapis.com) so buckets no longer collide on the cloud_assets unique key; previously they all shared storage.googleapis.com and overwrote each other. Live: 30 open buckets in ~5 min (e.g. estimator-dev, image-prod, update-media, mos-images S3; several GCS buckets), 11k+ objects indexed.
99 lines
3.8 KiB
Go
99 lines
3.8 KiB
Go
// Package metrics exposes lightweight Prometheus text-format metrics without
|
|
// pulling in a metrics dependency. Counters are plain atomics; gauges are
|
|
// callback functions so they always report current values.
|
|
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"runtime"
|
|
"sync/atomic"
|
|
)
|
|
|
|
type gauge struct {
|
|
name string
|
|
help string
|
|
extra string
|
|
fn func() float64
|
|
}
|
|
|
|
type Metrics struct {
|
|
CTCertificates atomic.Uint64
|
|
CTIdentifiers atomic.Uint64
|
|
|
|
DirectMatches atomic.Uint64
|
|
CNAMEMatches atomic.Uint64
|
|
|
|
DNSQueries atomic.Uint64
|
|
DNSCacheHits atomic.Uint64
|
|
DNSErrors atomic.Uint64
|
|
|
|
Candidates atomic.Uint64
|
|
|
|
DBRows atomic.Uint64
|
|
DBErrors atomic.Uint64
|
|
|
|
QueueDrops atomic.Uint64
|
|
QueueFull atomic.Uint64
|
|
MalformedCT atomic.Uint64
|
|
CTErrors atomic.Uint64
|
|
ProbeAttempts atomic.Uint64
|
|
ProbePublic atomic.Uint64
|
|
EntriesIndexed atomic.Uint64
|
|
|
|
BucketCandidates atomic.Uint64
|
|
BucketOpen atomic.Uint64
|
|
|
|
gauges []gauge
|
|
}
|
|
|
|
func New() *Metrics {
|
|
m := &Metrics{}
|
|
m.gauges = []gauge{
|
|
{"go_goroutines", "Number of goroutines", "", func() float64 { return float64(runtime.NumGoroutine()) }},
|
|
{"process_system_bytes", "Bytes of memory obtained from the OS", "", func() float64 {
|
|
var ms runtime.MemStats
|
|
runtime.ReadMemStats(&ms)
|
|
return float64(ms.Sys)
|
|
}},
|
|
}
|
|
return m
|
|
}
|
|
|
|
// SetQueueDepth registers a gauge reporting the live depth of a named queue.
|
|
func (m *Metrics) SetQueueDepth(name string, fn func() float64) {
|
|
m.gauges = append(m.gauges, gauge{"queue_depth", "Current depth of an internal queue", `{queue="` + name + `"}`, fn})
|
|
}
|
|
|
|
// Handler returns an http.Handler serving the Prometheus text exposition format.
|
|
func (m *Metrics) Handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
|
counter := func(name, help string, v uint64) {
|
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v)
|
|
}
|
|
counter("ct_certificates_total", "Certificates processed from CT logs", m.CTCertificates.Load())
|
|
counter("ct_identifiers_total", "DNS identifiers parsed from CT certificates", m.CTIdentifiers.Load())
|
|
counter("classifier_direct_matches_total", "Identifiers matching a provider directly", m.DirectMatches.Load())
|
|
counter("classifier_cname_matches_total", "CNAME targets matching a provider", m.CNAMEMatches.Load())
|
|
counter("dns_queries_total", "DNS queries performed", m.DNSQueries.Load())
|
|
counter("dns_cache_hits_total", "DNS lookups served from cache", m.DNSCacheHits.Load())
|
|
counter("dns_errors_total", "DNS lookup errors", m.DNSErrors.Load())
|
|
counter("candidates_total", "Provider candidates produced", m.Candidates.Load())
|
|
counter("db_rows_total", "Rows written to PostgreSQL", m.DBRows.Load())
|
|
counter("db_errors_total", "PostgreSQL write errors", m.DBErrors.Load())
|
|
counter("queue_drops_total", "Items dropped because a queue was full", m.QueueDrops.Load())
|
|
counter("queue_full_total", "Times a queue was observed full", m.QueueFull.Load())
|
|
counter("ct_malformed_entries_total", "Malformed CT log entries", m.MalformedCT.Load())
|
|
counter("ct_errors_total", "Non-fatal CT errors", m.CTErrors.Load())
|
|
counter("probe_attempts_total", "Optional reachability probes performed", m.ProbeAttempts.Load())
|
|
counter("probe_public_total", "Probes classified as public read", m.ProbePublic.Load())
|
|
counter("entries_indexed_total", "Metadata entries indexed from readable resources", m.EntriesIndexed.Load())
|
|
counter("bucket_candidates_total", "Candidate bucket names tested", m.BucketCandidates.Load())
|
|
counter("bucket_open_total", "Open (listable) buckets discovered by name enumeration", m.BucketOpen.Load())
|
|
for _, g := range m.gauges {
|
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s%s %g\n", g.name, g.help, g.name, g.name, g.extra, g.fn())
|
|
}
|
|
})
|
|
}
|