ctcloud: CertSpotter-backed CT cloud-resource monitor
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
This commit is contained in:
StasonJatham committed
1 parent
9e78a114ac
commit
e8f7d6bfdf
118 files changed
+16473
No files matched your search
@@ -0,0 +1,198 @@
|
||||
// Package classify maps a DNS hostname to a cloud/database/storage resource.
|
||||
//
|
||||
// The hot path is deliberately allocation-free: it never lowercases unless an
|
||||
// uppercase byte is present, never splits the hostname into a []string, never
|
||||
// uses regexp/JSON/reflection, and returns substrings of the input as the
|
||||
// region/resource. See BenchmarkClassify* for the 0 allocs/op assertion.
|
||||
package classify
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
const maxLabels = 64
|
||||
|
||||
// scanner holds the byte offsets of each dot-separated label of a hostname.
|
||||
// It is stack allocated by Classify and never heap allocated.
|
||||
type scanner struct {
|
||||
host string
|
||||
n int
|
||||
start [maxLabels]int32
|
||||
}
|
||||
|
||||
func newScanner(host string) scanner {
|
||||
var s scanner
|
||||
s.host = host
|
||||
start := int32(0)
|
||||
for i := 0; i < len(host); i++ {
|
||||
if host[i] == '.' {
|
||||
if s.n >= maxLabels {
|
||||
s.n = -1
|
||||
return s
|
||||
}
|
||||
s.start[s.n] = start
|
||||
s.n++
|
||||
start = int32(i + 1)
|
||||
}
|
||||
}
|
||||
if s.n >= maxLabels {
|
||||
s.n = -1
|
||||
return s
|
||||
}
|
||||
s.start[s.n] = start
|
||||
s.n++
|
||||
return s
|
||||
}
|
||||
|
||||
// label returns the i-th label (0 = leftmost). Callers must ensure i < n.
|
||||
func (s *scanner) label(i int) string {
|
||||
end := int32(len(s.host))
|
||||
if i+1 < s.n {
|
||||
end = s.start[i+1] - 1
|
||||
}
|
||||
return s.host[s.start[i]:end]
|
||||
}
|
||||
|
||||
func hasUpper(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] >= 'A' && s[i] <= 'Z' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func trimTrailingDots(s string) string {
|
||||
for len(s) > 0 && s[len(s)-1] == '.' {
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func validHost(host string) bool {
|
||||
if host == "" || len(host) > 253 {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(host); i++ {
|
||||
c := host[i]
|
||||
switch {
|
||||
case c >= 'a' && c <= 'z':
|
||||
case c >= '0' && c <= '9':
|
||||
case c == '-' || c == '.' || c == '_':
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Classify returns the provider resource described by host, if any.
|
||||
// The returned Match's string fields alias host.
|
||||
func Classify(host string) (model.Match, bool) {
|
||||
host = trimTrailingDots(host)
|
||||
if host == "" || len(host) > 253 {
|
||||
return model.Match{}, false
|
||||
}
|
||||
// Wildcard certificates never describe a concrete resource.
|
||||
if host[0] == '*' {
|
||||
return model.Match{}, false
|
||||
}
|
||||
if hasUpper(host) {
|
||||
host = strings.ToLower(host)
|
||||
}
|
||||
if !validHost(host) {
|
||||
return model.Match{}, false
|
||||
}
|
||||
|
||||
s := newScanner(host)
|
||||
if s.n < 2 {
|
||||
return model.Match{}, false
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasSuffix(host, ".amazonaws.com"):
|
||||
return classifyAWS(&s)
|
||||
case strings.HasSuffix(host, ".core.windows.net"):
|
||||
return classifyAzureCore(&s)
|
||||
case strings.HasSuffix(host, ".database.windows.net"):
|
||||
return classifyAzureDatabase(&s)
|
||||
case strings.HasSuffix(host, ".database.azure.com"):
|
||||
return classifyAzureFlexibleServer(&s)
|
||||
case strings.HasSuffix(host, ".documents.azure.com"):
|
||||
return classifyAzureDocuments(&s)
|
||||
case strings.HasSuffix(host, ".cosmos.azure.com"):
|
||||
return classifyAzureCosmos(&s)
|
||||
case strings.HasSuffix(host, ".cache.windows.net"):
|
||||
return classifyAzureRedis(&s)
|
||||
case strings.HasSuffix(host, ".redis.azure.net"):
|
||||
return classifyAzureManagedRedis(&s)
|
||||
case strings.HasSuffix(host, ".search.windows.net"):
|
||||
return classifyAzureSearch(&s)
|
||||
case strings.HasSuffix(host, ".storage.googleapis.com"):
|
||||
return classifyGCS(&s)
|
||||
case strings.HasSuffix(host, ".firebaseio.com"):
|
||||
return classifyFirebaseIO(&s)
|
||||
case strings.HasSuffix(host, ".firebasedatabase.app"):
|
||||
return classifyFirebaseDB(&s)
|
||||
case strings.HasSuffix(host, ".appspot.com"):
|
||||
return classifyAppspot(&s)
|
||||
case strings.HasSuffix(host, ".r2.cloudflarestorage.com"):
|
||||
return classifyR2(&s)
|
||||
case strings.HasSuffix(host, ".r2.dev"):
|
||||
return classifyR2Dev(&s)
|
||||
case strings.HasSuffix(host, ".digitaloceanspaces.com"):
|
||||
return classifyDigitalOceanSpaces(&s)
|
||||
case strings.HasSuffix(host, ".ondigitalocean.com"):
|
||||
return classifyDigitalOceanDB(&s)
|
||||
case strings.HasSuffix(host, ".supabase.co"):
|
||||
return classifySupabase(&s)
|
||||
case strings.HasSuffix(host, ".supabase.com"):
|
||||
return classifySupabasePooler(&s)
|
||||
case strings.HasSuffix(host, ".customer-oci.com"):
|
||||
return classifyOCIResource(&s)
|
||||
case strings.HasSuffix(host, ".oraclecloud.com"):
|
||||
return classifyOCIGeneric(&s)
|
||||
}
|
||||
return model.Match{}, false
|
||||
}
|
||||
|
||||
// ShouldEnrich reports whether a non-matching hostname is worth a DNS/CNAME
|
||||
// lookup. It rejects wildcards, the certspotter unparsable placeholder, single
|
||||
// labels and obvious provider service apexes.
|
||||
func ShouldEnrich(host string) bool {
|
||||
host = trimTrailingDots(host)
|
||||
if len(host) < 3 || host[0] == '*' {
|
||||
return false
|
||||
}
|
||||
if host[0] == '<' { // certspotter's "<unparsable>" placeholder
|
||||
return false
|
||||
}
|
||||
if !strings.Contains(host, ".") {
|
||||
return false
|
||||
}
|
||||
if hasUpper(host) {
|
||||
host = strings.ToLower(host)
|
||||
}
|
||||
// Provider service apexes never yield a customer resource via CNAME.
|
||||
switch {
|
||||
case host == "amazonaws.com", strings.HasSuffix(host, ".amazonaws.com"):
|
||||
return false
|
||||
case host == "core.windows.net", strings.HasSuffix(host, ".windows.net"):
|
||||
return false
|
||||
case host == "azure.com", strings.HasSuffix(host, ".azure.com"):
|
||||
return false
|
||||
case host == "googleapis.com", strings.HasSuffix(host, ".googleapis.com"):
|
||||
return false
|
||||
case host == "cloudflarestorage.com", host == "r2.dev":
|
||||
return false
|
||||
case strings.HasSuffix(host, ".digitaloceanspaces.com"):
|
||||
return false
|
||||
case strings.HasSuffix(host, ".supabase.co"), strings.HasSuffix(host, ".supabase.com"):
|
||||
return false
|
||||
case strings.HasSuffix(host, ".oraclecloud.com"), strings.HasSuffix(host, ".customer-oci.com"):
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in new issue
Block a user