ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
This commit is contained in:
StasonJatham committed
1 parent
9e78a114ac
commit
e8f7d6bfdf
118 files changed
+16473
No files matched your search
@@ -0,0 +1,99 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type cacheEntry[V any] struct {
|
||||
val V
|
||||
exp time.Time
|
||||
}
|
||||
|
||||
type shard[V any] struct {
|
||||
mu sync.Mutex
|
||||
m map[string]cacheEntry[V]
|
||||
}
|
||||
|
||||
// TTLCache is a sharded, memory-bounded, in-memory TTL cache. It is used for
|
||||
// positive DNS results, negative DNS results, CNAME targets and hostname dedup.
|
||||
//
|
||||
// ponytail: once a shard is full, an arbitrary entry is evicted rather than the
|
||||
// least-recently-used one. Swap in an LRU only if the hit rate actually matters.
|
||||
type TTLCache[V any] struct {
|
||||
shards []shard[V]
|
||||
mask uint64
|
||||
maxShard int
|
||||
}
|
||||
|
||||
func NewTTLCache[V any](maxEntries, shards int) *TTLCache[V] {
|
||||
if shards < 1 {
|
||||
shards = 1
|
||||
}
|
||||
n := 1
|
||||
for n < shards {
|
||||
n <<= 1
|
||||
}
|
||||
maxShard := maxEntries / n
|
||||
if maxShard < 1 {
|
||||
maxShard = 1
|
||||
}
|
||||
c := &TTLCache[V]{mask: uint64(n - 1), maxShard: maxShard}
|
||||
c.shards = make([]shard[V], n)
|
||||
for i := range c.shards {
|
||||
c.shards[i].m = make(map[string]cacheEntry[V], maxShard)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func hashString(s string) uint64 {
|
||||
const prime = 1099511628211
|
||||
h := uint64(14695981039346656037)
|
||||
for i := 0; i < len(s); i++ {
|
||||
h ^= uint64(s[i])
|
||||
h *= prime
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (c *TTLCache[V]) Get(key string) (V, bool) {
|
||||
s := &c.shards[hashString(key)&c.mask]
|
||||
s.mu.Lock()
|
||||
e, ok := s.m[key]
|
||||
if ok && time.Now().Before(e.exp) {
|
||||
s.mu.Unlock()
|
||||
return e.val, true
|
||||
}
|
||||
if ok {
|
||||
delete(s.m, key)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
var zero V
|
||||
return zero, false
|
||||
}
|
||||
|
||||
func (c *TTLCache[V]) Set(key string, val V, ttl time.Duration) {
|
||||
if ttl <= 0 {
|
||||
return
|
||||
}
|
||||
s := &c.shards[hashString(key)&c.mask]
|
||||
s.mu.Lock()
|
||||
if len(s.m) >= c.maxShard {
|
||||
for k := range s.m {
|
||||
delete(s.m, k)
|
||||
break
|
||||
}
|
||||
}
|
||||
s.m[key] = cacheEntry[V]{val: val, exp: time.Now().Add(ttl)}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *TTLCache[V]) Len() int {
|
||||
total := 0
|
||||
for i := range c.shards {
|
||||
c.shards[i].mu.Lock()
|
||||
total += len(c.shards[i].m)
|
||||
c.shards[i].mu.Unlock()
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ServerPool load-balances DNS lookups across multiple resolvers using
|
||||
// round-robin, and fails over to the next server on a timeout or server error.
|
||||
// Definitive answers (NXDOMAIN) are returned immediately and are not retried on
|
||||
// every server.
|
||||
// defaultAttemptTimeout bounds each individual server attempt so a dead
|
||||
// resolver costs a couple of seconds before failover, not the entire lookup
|
||||
// budget.
|
||||
const defaultAttemptTimeout = 2 * time.Second
|
||||
|
||||
type ServerPool struct {
|
||||
resolvers []Resolver
|
||||
next atomic.Uint64
|
||||
attemptTimeout time.Duration
|
||||
}
|
||||
|
||||
func (p *ServerPool) withAttemptTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
t := p.attemptTimeout
|
||||
if t <= 0 {
|
||||
t = defaultAttemptTimeout
|
||||
}
|
||||
return context.WithTimeout(ctx, t)
|
||||
}
|
||||
|
||||
// NewServerPool builds a pool from "host", "host:port" or "system" entries.
|
||||
// An empty list uses the system resolver.
|
||||
func NewServerPool(servers []string) *ServerPool {
|
||||
var resolvers []Resolver
|
||||
for _, s := range servers {
|
||||
s = strings.TrimSpace(s)
|
||||
switch {
|
||||
case s == "":
|
||||
continue
|
||||
case s == "system" || s == "local":
|
||||
return &ServerPool{resolvers: []Resolver{&GoResolver{R: net.DefaultResolver}}}
|
||||
default:
|
||||
if !strings.Contains(s, ":") {
|
||||
s += ":53"
|
||||
}
|
||||
resolvers = append(resolvers, &GoResolver{R: newServerResolver(s)})
|
||||
}
|
||||
}
|
||||
if len(resolvers) == 0 {
|
||||
resolvers = append(resolvers, &GoResolver{R: net.DefaultResolver})
|
||||
}
|
||||
return &ServerPool{resolvers: resolvers}
|
||||
}
|
||||
|
||||
func newServerResolver(server string) *net.Resolver {
|
||||
return &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
if network != "tcp" {
|
||||
network = "udp"
|
||||
}
|
||||
d := net.Dialer{Timeout: 5 * time.Second}
|
||||
return d.DialContext(ctx, network, server)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ServerPool) pick() int {
|
||||
return int((p.next.Add(1) - 1) % uint64(len(p.resolvers)))
|
||||
}
|
||||
|
||||
// definitive reports whether err is a real DNS answer (NXDOMAIN) that should not
|
||||
// be retried against other servers.
|
||||
func definitive(err error) bool {
|
||||
var dnsErr *net.DNSError
|
||||
return errors.As(err, &dnsErr) && dnsErr.IsNotFound
|
||||
}
|
||||
|
||||
func (p *ServerPool) LookupCNAME(ctx context.Context, host string) (string, error) {
|
||||
n := len(p.resolvers)
|
||||
start := p.pick()
|
||||
var last error
|
||||
for i := 0; i < n; i++ {
|
||||
actx, cancel := p.withAttemptTimeout(ctx)
|
||||
cname, err := p.resolvers[(start+i)%n].LookupCNAME(actx, host)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return cname, nil
|
||||
}
|
||||
if definitive(err) {
|
||||
return "", err
|
||||
}
|
||||
last = err
|
||||
}
|
||||
return "", last
|
||||
}
|
||||
|
||||
func (p *ServerPool) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
|
||||
n := len(p.resolvers)
|
||||
start := p.pick()
|
||||
var last error
|
||||
for i := 0; i < n; i++ {
|
||||
actx, cancel := p.withAttemptTimeout(ctx)
|
||||
ips, err := p.resolvers[(start+i)%n].LookupIP(actx, host)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return ips, nil
|
||||
}
|
||||
if definitive(err) {
|
||||
return nil, err
|
||||
}
|
||||
last = err
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeLookup struct {
|
||||
calls atomic.Int32
|
||||
cnameErr error
|
||||
ipErr error
|
||||
}
|
||||
|
||||
func (f *fakeLookup) LookupCNAME(ctx context.Context, host string) (string, error) {
|
||||
f.calls.Add(1)
|
||||
if f.cnameErr != nil {
|
||||
return "", f.cnameErr
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
func (f *fakeLookup) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
|
||||
f.calls.Add(1)
|
||||
if f.ipErr != nil {
|
||||
return nil, f.ipErr
|
||||
}
|
||||
return []net.IP{net.ParseIP("1.2.3.4")}, nil
|
||||
}
|
||||
|
||||
func TestServerPoolRoundRobin(t *testing.T) {
|
||||
a, b := &fakeLookup{}, &fakeLookup{}
|
||||
p := &ServerPool{resolvers: []Resolver{a, b}}
|
||||
for i := 0; i < 4; i++ {
|
||||
if _, err := p.LookupIP(context.Background(), "example.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if a.calls.Load() != 2 || b.calls.Load() != 2 {
|
||||
t.Fatalf("round robin calls: a=%d b=%d, want 2/2", a.calls.Load(), b.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerPoolFailover(t *testing.T) {
|
||||
a := &fakeLookup{ipErr: &net.DNSError{Err: "timeout", IsTimeout: true}}
|
||||
b := &fakeLookup{}
|
||||
p := &ServerPool{resolvers: []Resolver{a, b}}
|
||||
ips, err := p.LookupIP(context.Background(), "example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ips) != 1 || a.calls.Load() != 1 {
|
||||
t.Fatalf("expected failover to second server, a.calls=%d", a.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerPoolNoFailoverOnNXDOMAIN(t *testing.T) {
|
||||
a := &fakeLookup{ipErr: &net.DNSError{Err: "no such host", IsNotFound: true}}
|
||||
b := &fakeLookup{}
|
||||
p := &ServerPool{resolvers: []Resolver{a, b}}
|
||||
if _, err := p.LookupIP(context.Background(), "missing.example.com"); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if b.calls.Load() != 0 {
|
||||
t.Fatalf("NXDOMAIN must not fail over; b.calls=%d", b.calls.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// Package dns provides a bounded, cached DNS enricher that follows CNAME chains
|
||||
// with loop detection.
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/metrics"
|
||||
)
|
||||
|
||||
// ErrCNAMELoop is returned when a CNAME chain revisits a host.
|
||||
var ErrCNAMELoop = errors.New("cname loop detected")
|
||||
|
||||
// Resolver is the subset of DNS operations the enricher needs.
|
||||
type Resolver interface {
|
||||
LookupCNAME(ctx context.Context, host string) (string, error)
|
||||
LookupIP(ctx context.Context, host string) ([]net.IP, error)
|
||||
}
|
||||
|
||||
// Result is a resolved hostname: its CNAME chain (chain[0] is the queried host)
|
||||
// and any A/AAAA addresses.
|
||||
type Result struct {
|
||||
Chain []string
|
||||
IPs []net.IP
|
||||
}
|
||||
|
||||
// GoResolver uses the standard library resolver.
|
||||
//
|
||||
// ponytail: net.Resolver.LookupCNAME returns the final canonical name, so the
|
||||
// chain has at most two entries and intermediate CNAME hops are not observed.
|
||||
// Swap in miekg/dns if the full chain is ever needed.
|
||||
type GoResolver struct {
|
||||
R *net.Resolver
|
||||
}
|
||||
|
||||
func NewGoResolver() *GoResolver {
|
||||
return &GoResolver{R: net.DefaultResolver}
|
||||
}
|
||||
|
||||
func (g *GoResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
|
||||
return g.R.LookupCNAME(ctx, host)
|
||||
}
|
||||
|
||||
func (g *GoResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
|
||||
return g.R.LookupIP(ctx, "ip", host)
|
||||
}
|
||||
|
||||
// Follow walks a CNAME chain up to maxDepth hops, detecting loops.
|
||||
func Follow(ctx context.Context, r Resolver, host string, maxDepth int) ([]string, error) {
|
||||
if maxDepth < 1 {
|
||||
maxDepth = 1
|
||||
}
|
||||
chain := []string{host}
|
||||
cur := host
|
||||
for i := 0; i < maxDepth; i++ {
|
||||
next, err := r.LookupCNAME(ctx, cur)
|
||||
if err != nil {
|
||||
return chain, err
|
||||
}
|
||||
next = strings.TrimSuffix(next, ".")
|
||||
if next == "" || next == cur {
|
||||
break
|
||||
}
|
||||
for _, seen := range chain {
|
||||
if seen == next {
|
||||
return chain, ErrCNAMELoop
|
||||
}
|
||||
}
|
||||
chain = append(chain, next)
|
||||
cur = next
|
||||
}
|
||||
return chain, nil
|
||||
}
|
||||
|
||||
// Enricher resolves hostnames with caching and bounded CNAME depth. It is safe
|
||||
// for concurrent use.
|
||||
type Enricher struct {
|
||||
Resolver Resolver
|
||||
Cache *TTLCache[Result]
|
||||
PositiveTTL time.Duration
|
||||
NegativeTTL time.Duration
|
||||
Timeout time.Duration
|
||||
MaxDepth int
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// Resolve returns the result for host. ok is false when the lookup failed;
|
||||
// failures (including "no such host") are negatively cached for NegativeTTL.
|
||||
func (e *Enricher) Resolve(ctx context.Context, host string) (Result, bool) {
|
||||
if e.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, e.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
if e.Cache != nil {
|
||||
if res, ok := e.Cache.Get(host); ok {
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.DNSCacheHits.Add(1)
|
||||
}
|
||||
return res, len(res.Chain) > 0 || len(res.IPs) > 0
|
||||
}
|
||||
}
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.DNSQueries.Add(1)
|
||||
}
|
||||
|
||||
chain, cnameErr := Follow(ctx, e.Resolver, host, e.MaxDepth)
|
||||
if cnameErr != nil && len(chain) == 1 {
|
||||
// A definitive NXDOMAIN has no A/AAAA to fetch either, so skip the
|
||||
// second lookup. NXDOMAIN is a valid answer, not an infrastructure error.
|
||||
if e.Metrics != nil && !definitive(cnameErr) {
|
||||
e.Metrics.DNSErrors.Add(1)
|
||||
}
|
||||
if e.Cache != nil {
|
||||
e.Cache.Set(host, Result{}, e.NegativeTTL)
|
||||
}
|
||||
return Result{}, false
|
||||
}
|
||||
res := Result{Chain: chain}
|
||||
if ips, err := e.Resolver.LookupIP(ctx, host); err == nil {
|
||||
res.IPs = ips
|
||||
}
|
||||
if e.Cache != nil {
|
||||
e.Cache.Set(host, res, e.PositiveTTL)
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeResolver struct {
|
||||
cnames map[string]string
|
||||
ips map[string][]net.IP
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
if c, ok := f.cnames[host]; ok {
|
||||
return c, nil
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
func (f *fakeResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.ips[host], nil
|
||||
}
|
||||
|
||||
func TestFollowChain(t *testing.T) {
|
||||
r := &fakeResolver{cnames: map[string]string{
|
||||
"a.example.com": "b.example.com",
|
||||
"b.example.com": "c.s3.amazonaws.com",
|
||||
"c.s3.amazonaws.com": "c.s3.amazonaws.com",
|
||||
}}
|
||||
chain, err := Follow(context.Background(), r, "a.example.com", 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"a.example.com", "b.example.com", "c.s3.amazonaws.com"}
|
||||
if len(chain) != len(want) {
|
||||
t.Fatalf("chain = %v, want %v", chain, want)
|
||||
}
|
||||
for i := range want {
|
||||
if chain[i] != want[i] {
|
||||
t.Fatalf("chain = %v, want %v", chain, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFollowLoop(t *testing.T) {
|
||||
r := &fakeResolver{cnames: map[string]string{
|
||||
"a.example.com": "b.example.com",
|
||||
"b.example.com": "a.example.com",
|
||||
}}
|
||||
chain, err := Follow(context.Background(), r, "a.example.com", 8)
|
||||
if !errors.Is(err, ErrCNAMELoop) {
|
||||
t.Fatalf("err = %v, want ErrCNAMELoop", err)
|
||||
}
|
||||
if len(chain) < 2 {
|
||||
t.Fatalf("chain = %v, want at least 2 entries", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFollowDepth(t *testing.T) {
|
||||
r := &fakeResolver{cnames: map[string]string{
|
||||
"h1.example.com": "h2.example.com",
|
||||
"h2.example.com": "h3.example.com",
|
||||
"h3.example.com": "h4.example.com",
|
||||
}}
|
||||
chain, err := Follow(context.Background(), r, "h1.example.com", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(chain) != 3 {
|
||||
t.Fatalf("chain = %v, want 3 entries (host + 2 hops)", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnricherCaches(t *testing.T) {
|
||||
r := &fakeResolver{
|
||||
cnames: map[string]string{"a.example.com": "b.s3.amazonaws.com"},
|
||||
ips: map[string][]net.IP{"a.example.com": {net.ParseIP("1.2.3.4")}},
|
||||
}
|
||||
e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
|
||||
if _, ok := e.Resolve(context.Background(), "a.example.com"); !ok {
|
||||
t.Fatal("first resolve failed")
|
||||
}
|
||||
r.cnames = nil // a cached lookup must not hit the resolver
|
||||
res, ok := e.Resolve(context.Background(), "a.example.com")
|
||||
if !ok || len(res.IPs) != 1 {
|
||||
t.Fatalf("cached resolve = %+v ok=%v", res, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnricherNegativeCache(t *testing.T) {
|
||||
r := &fakeResolver{err: &net.DNSError{Err: "no such host", IsNotFound: true}}
|
||||
e := &Enricher{Resolver: r, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
|
||||
if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
if _, ok := e.Resolve(context.Background(), "missing.example.com"); ok {
|
||||
t.Fatal("expected cached failure")
|
||||
}
|
||||
}
|
||||
|
||||
type ctxResolver struct{}
|
||||
|
||||
func (ctxResolver) LookupCNAME(ctx context.Context, host string) (string, error) {
|
||||
return "", ctx.Err()
|
||||
}
|
||||
func (ctxResolver) LookupIP(ctx context.Context, host string) ([]net.IP, error) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
func TestResolveContextCancellation(t *testing.T) {
|
||||
e := &Enricher{Resolver: ctxResolver{}, Cache: NewTTLCache[Result](16, 2), PositiveTTL: time.Minute, NegativeTTL: time.Minute, MaxDepth: 8}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, ok := e.Resolve(ctx, "example.com"); ok {
|
||||
t.Fatal("expected cancellation to fail resolution")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheBoundAndExpiry(t *testing.T) {
|
||||
c := NewTTLCache[int](8, 2)
|
||||
for i := 0; i < 100; i++ {
|
||||
c.Set(string(rune('a'+i%26))+string(rune('0'+i%10)), i, time.Minute)
|
||||
}
|
||||
if c.Len() > 8 {
|
||||
t.Fatalf("cache len = %d, want <= 8", c.Len())
|
||||
}
|
||||
c.Set("k", 1, time.Millisecond)
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if _, ok := c.Get("k"); ok {
|
||||
t.Fatal("expected expired entry to be gone")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user