over-engineering audit: remove dead code, speculative config, duplicate queues

- delete embedded community resolver snapshot (6,760 lines) + EmbeddedServers +
  DNS_SERVERS=fresh; curated resolvers are the default. Simplify Warm (no
  sampling).
- one generic internal/sample.Queue[T] replaces three near-identical
  random-sample queues (probe, pipeline, discover).
- delete unreachable index.listOpenSearch (OpenSearch is never listable).
- delete classifyCloudFront (its match is always discarded, !ResourceSpecific).
- delete probe.Probe wrapper + Describe; tests use ProbeDetailed.
- delete store.Pool(), model.SourceDNSAlias, discover.QueueLen.
- drop dead config/no-op knobs: pipeline DirectWorkers, discover MaxPerDomain,
  discover S3/GCS/DO flags, BUCKET_SCAN_DO.
~-7,000 lines (mostly the resolver snapshot), 0 deps removed, tests green.
This commit is contained in:
StasonJatham committed 2026-09-22 17:12:30 +02:00
1 parent 81e53f8755
commit dc7bdae99d
22 files changed
+150 -7202

No files matched your search

+7 -67
View File
@@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"io"
mathrand "math/rand/v2"
"net"
"net/http"
"net/url"
@@ -32,6 +31,7 @@ import (
"tea.karlcom.de/karl/bucketeer/internal/index"
"tea.karlcom.de/karl/bucketeer/internal/metrics"
"tea.karlcom.de/karl/bucketeer/internal/model"
"tea.karlcom.de/karl/bucketeer/internal/sample"
)
// Result is the outcome of probing one candidate.
@@ -75,61 +75,12 @@ type Config struct {
type Prober struct {
cfg Config
sample *sampleQueue
sample *sample.Queue[model.Candidate]
client *http.Client
stop chan struct{}
wg sync.WaitGroup
}
// sampleQueue is a memory-bounded queue that keeps a random sample of recent
// submissions rather than a FIFO, so one high-volume hostname template cannot
// starve every other asset type. It is not on the certificate hot path.
type sampleQueue struct {
mu sync.Mutex
items []model.Candidate
cap int
notify chan struct{}
}
func newSampleQueue(capacity int) *sampleQueue {
return &sampleQueue{cap: capacity, notify: make(chan struct{}, 1)}
}
func (q *sampleQueue) push(c model.Candidate) {
q.mu.Lock()
if len(q.items) < q.cap {
q.items = append(q.items, c)
} else {
q.items[mathrand.IntN(len(q.items))] = c
}
q.mu.Unlock()
select {
case q.notify <- struct{}{}:
default:
}
}
func (q *sampleQueue) len() int {
q.mu.Lock()
defer q.mu.Unlock()
return len(q.items)
}
func (q *sampleQueue) pop() (model.Candidate, bool) {
q.mu.Lock()
n := len(q.items)
if n == 0 {
q.mu.Unlock()
return model.Candidate{}, false
}
i := mathrand.IntN(n)
c := q.items[i]
q.items[i] = q.items[n-1]
q.items = q.items[:n-1]
q.mu.Unlock()
return c, true
}
func New(cfg Config) *Prober {
if cfg.Workers <= 0 {
cfg.Workers = 4
@@ -142,7 +93,7 @@ func New(cfg Config) *Prober {
}
return &Prober{
cfg: cfg,
sample: newSampleQueue(cfg.Queue),
sample: sample.New[model.Candidate](cfg.Queue),
stop: make(chan struct{}),
client: &http.Client{
Timeout: cfg.Timeout,
@@ -178,7 +129,7 @@ func (p *Prober) Submit(c model.Candidate) {
if p.cfg.Metrics != nil {
p.cfg.Metrics.ProbeAttempts.Add(1)
}
p.sample.push(c)
p.sample.Push(c)
}
// Close stops the workers after any in-flight probe. It must be called before
@@ -188,7 +139,7 @@ func (p *Prober) Close() { close(p.stop) }
func (p *Prober) Wait() { p.wg.Wait() }
// QueueLen reports the current probe sample depth.
func (p *Prober) QueueLen() int { return p.sample.len() }
func (p *Prober) QueueLen() int { return p.sample.Len() }
func (p *Prober) worker(ctx context.Context) {
defer p.wg.Done()
@@ -198,10 +149,10 @@ func (p *Prober) worker(ctx context.Context) {
return
case <-p.stop:
return
case <-p.sample.notify:
case <-p.sample.Notify():
}
for {
c, ok := p.sample.pop()
c, ok := p.sample.Pop()
if !ok {
break
}
@@ -225,12 +176,6 @@ func (p *Prober) worker(ctx context.Context) {
}
}
// Probe is the (exposure, detail) convenience view.
func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
r := p.ProbeDetailed(ctx, c)
return r.Exposure, r.Detail
}
// ProbeDetailed runs the vendor-appropriate check and reports capabilities.
func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result {
// Deep, read-only protocol checks take precedence for database services.
@@ -469,8 +414,3 @@ func classifyError(err error) (model.ExposureState, string) {
}
return model.ExposureUnknown, err.Error()
}
// Describe is a helper for logs/tests.
func Describe(s model.ExposureState) string {
return fmt.Sprintf("exposure=%s", s)
}
+15 -15
View File
@@ -38,11 +38,11 @@ func TestProbeStorageStatuses(t *testing.T) {
for _, tc := range cases {
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, statusHandler(tc.status))
got, detail := p.Probe(context.Background(), model.Candidate{
r := p.ProbeDetailed(context.Background(), model.Candidate{
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
})
if got != tc.want {
t.Errorf("status %d: got %s (%s), want %s", tc.status, got, detail, tc.want)
if r.Exposure != tc.want {
t.Errorf("status %d: got %s (%s), want %s", tc.status, r.Exposure, r.Detail, tc.want)
}
}
}
@@ -54,29 +54,29 @@ func TestProbeS3RegionRedirect(t *testing.T) {
})
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, h)
got, detail := p.Probe(context.Background(), model.Candidate{
r := p.ProbeDetailed(context.Background(), model.Candidate{
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
})
if got != model.ExposureReachable || !strings.Contains(detail, "eu-central-1") {
t.Fatalf("got %s (%s), want reachable with region", got, detail)
if r.Exposure != model.ExposureReachable || !strings.Contains(r.Detail, "eu-central-1") {
t.Fatalf("got %s (%s), want reachable with region", r.Exposure, r.Detail)
}
}
func TestProbeFirebase(t *testing.T) {
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, statusHandler(http.StatusUnauthorized))
got, _ := p.Probe(context.Background(), model.Candidate{
r := p.ProbeDetailed(context.Background(), model.Candidate{
Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
})
if got != model.ExposureAuthRequired {
t.Fatalf("got %s, want auth_required", got)
if r.Exposure != model.ExposureAuthRequired {
t.Fatalf("got %s, want auth_required", r.Exposure)
}
host2 := serverHost(t, statusHandler(http.StatusOK))
got, _ = p.Probe(context.Background(), model.Candidate{
r = p.ProbeDetailed(context.Background(), model.Candidate{
Hostname: host2, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
})
if got != model.ExposurePublicRead {
t.Fatalf("got %s, want public_read", got)
if r.Exposure != model.ExposurePublicRead {
t.Fatalf("got %s, want public_read", r.Exposure)
}
}
@@ -90,11 +90,11 @@ func TestProbeAzureContainerDiscovery(t *testing.T) {
})
p := New(Config{Workers: 1, Timeout: time.Second})
host := serverHost(t, h)
state, detail := p.Probe(context.Background(), model.Candidate{
r := p.ProbeDetailed(context.Background(), model.Candidate{
Hostname: host, Resource: "acct", Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
})
if state != model.ExposurePublicRead || !strings.Contains(detail, "images") {
t.Fatalf("got %s (%s), want public_read via container 'images'", state, detail)
if r.Exposure != model.ExposurePublicRead || !strings.Contains(r.Detail, "images") {
t.Fatalf("got %s (%s), want public_read via container 'images'", r.Exposure, r.Detail)
}
}