Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
196 lines
4.2 KiB
Go
196 lines
4.2 KiB
Go
// Copyright (C) 2025 Opsmate, Inc.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla
|
|
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
|
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
//
|
|
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
|
// See the Mozilla Public License for details.
|
|
|
|
package sequencer
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
mathrand "math/rand/v2"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSequencerBasic(t *testing.T) {
|
|
ctx := context.Background()
|
|
seq := New[uint64](0, 100)
|
|
go func() {
|
|
for i := range uint64(10_000) {
|
|
err := seq.Add(ctx, i, &i)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
|
|
}
|
|
}
|
|
}()
|
|
|
|
for i := range uint64(10_000) {
|
|
next, err := seq.Next(ctx)
|
|
if err != nil {
|
|
t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
|
|
}
|
|
if *next != i {
|
|
t.Fatalf("%d: got unexpected value %d", i, *next)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSequencerNonZeroStart(t *testing.T) {
|
|
ctx := context.Background()
|
|
seq := New[uint64](10, 100)
|
|
go func() {
|
|
for i := range uint64(10_000) {
|
|
err := seq.Add(ctx, i+10, &i)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
|
|
}
|
|
}
|
|
}()
|
|
|
|
for i := range uint64(10_000) {
|
|
next, err := seq.Next(ctx)
|
|
if err != nil {
|
|
t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
|
|
}
|
|
if *next != i {
|
|
t.Fatalf("%d: got unexpected value %d", i, *next)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSequencerCapacity1(t *testing.T) {
|
|
ctx := context.Background()
|
|
seq := New[uint64](0, 1)
|
|
go func() {
|
|
for i := range uint64(10_000) {
|
|
err := seq.Add(ctx, i, &i)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
|
|
}
|
|
}
|
|
}()
|
|
|
|
for i := range uint64(10_000) {
|
|
next, err := seq.Next(ctx)
|
|
if err != nil {
|
|
t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
|
|
}
|
|
if *next != i {
|
|
t.Fatalf("%d: got unexpected value %d", i, *next)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSequencerTimeout(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
seq := New[uint64](0, 10_000)
|
|
go func() {
|
|
var i uint64
|
|
for {
|
|
newI := i
|
|
err := seq.Add(ctx, i, &newI)
|
|
if err != nil {
|
|
break
|
|
}
|
|
i++
|
|
}
|
|
}()
|
|
|
|
var i uint64
|
|
for {
|
|
next, err := seq.Next(ctx)
|
|
if err != nil {
|
|
break
|
|
}
|
|
if *next != i {
|
|
t.Fatalf("%d: got unexpected value %d", i, *next)
|
|
}
|
|
i++
|
|
}
|
|
}
|
|
|
|
func TestSequencerOutOfOrder(t *testing.T) {
|
|
ctx := context.Background()
|
|
seq := New[uint64](0, 100)
|
|
ch := make(chan uint64)
|
|
go func() {
|
|
for i := range uint64(10_000) {
|
|
ch <- i
|
|
}
|
|
}()
|
|
for range 4 {
|
|
go func() {
|
|
for i := range ch {
|
|
time.Sleep(mathrand.N(10 * time.Millisecond))
|
|
//t.Logf("seq.Add %d", i)
|
|
err := seq.Add(ctx, i, &i)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
for i := range uint64(10_000) {
|
|
next, err := seq.Next(ctx)
|
|
if err != nil {
|
|
t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
|
|
}
|
|
if *next != i {
|
|
t.Fatalf("%d: got unexpected value %d", i, *next)
|
|
}
|
|
//t.Logf("seq.Next %d", i)
|
|
}
|
|
}
|
|
|
|
func TestSequencerOutOfOrderReserve(t *testing.T) {
|
|
ctx := context.Background()
|
|
seq := New[uint64](0, 10)
|
|
ch := make(chan uint64)
|
|
go func() {
|
|
for i := range uint64(10_000) {
|
|
ch <- i
|
|
}
|
|
}()
|
|
ch2 := make(chan uint64)
|
|
for job := range 4 {
|
|
go func() {
|
|
for i := range ch {
|
|
time.Sleep(mathrand.N(11 * time.Duration(job+1) * time.Millisecond))
|
|
err := seq.Reserve(ctx, i)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("%d: seq.Reserve returned unexpected error %v", i, err))
|
|
}
|
|
ch2 <- i
|
|
}
|
|
}()
|
|
}
|
|
for range 4 {
|
|
go func() {
|
|
for i := range ch2 {
|
|
time.Sleep(mathrand.N(7 * time.Millisecond))
|
|
t.Logf("seq.Add %d", i)
|
|
err := seq.Add(ctx, i, &i)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
for i := range uint64(10_000) {
|
|
next, err := seq.Next(ctx)
|
|
if err != nil {
|
|
t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
|
|
}
|
|
if *next != i {
|
|
t.Fatalf("%d: got unexpected value %d", i, *next)
|
|
}
|
|
t.Logf("seq.Next %d", i)
|
|
}
|
|
}
|