over-engineering audit: remove dead code, speculative config, duplicate queues
- delete embedded community resolver snapshot (6,760 lines) + EmbeddedServers + DNS_SERVERS=fresh; curated resolvers are the default. Simplify Warm (no sampling). - one generic internal/sample.Queue[T] replaces three near-identical random-sample queues (probe, pipeline, discover). - delete unreachable index.listOpenSearch (OpenSearch is never listable). - delete classifyCloudFront (its match is always discarded, !ResourceSpecific). - delete probe.Probe wrapper + Describe; tests use ProbeDetailed. - delete store.Pool(), model.SourceDNSAlias, discover.QueueLen. - drop dead config/no-op knobs: pipeline DirectWorkers, discover MaxPerDomain, discover S3/GCS/DO flags, BUCKET_SCAN_DO. ~-7,000 lines (mostly the resolver snapshot), 0 deps removed, tests green.
This commit is contained in:
1 parent
81e53f8755
commit
dc7bdae99d
22 files changed
+150
-7202
No files matched your search
+7
-67
@@ -21,7 +21,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
mathrand "math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -32,6 +31,7 @@ import (
|
||||
"tea.karlcom.de/karl/bucketeer/internal/index"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/metrics"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/sample"
|
||||
)
|
||||
|
||||
// Result is the outcome of probing one candidate.
|
||||
@@ -75,61 +75,12 @@ type Config struct {
|
||||
|
||||
type Prober struct {
|
||||
cfg Config
|
||||
sample *sampleQueue
|
||||
sample *sample.Queue[model.Candidate]
|
||||
client *http.Client
|
||||
stop chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// sampleQueue is a memory-bounded queue that keeps a random sample of recent
|
||||
// submissions rather than a FIFO, so one high-volume hostname template cannot
|
||||
// starve every other asset type. It is not on the certificate hot path.
|
||||
type sampleQueue struct {
|
||||
mu sync.Mutex
|
||||
items []model.Candidate
|
||||
cap int
|
||||
notify chan struct{}
|
||||
}
|
||||
|
||||
func newSampleQueue(capacity int) *sampleQueue {
|
||||
return &sampleQueue{cap: capacity, notify: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
func (q *sampleQueue) push(c model.Candidate) {
|
||||
q.mu.Lock()
|
||||
if len(q.items) < q.cap {
|
||||
q.items = append(q.items, c)
|
||||
} else {
|
||||
q.items[mathrand.IntN(len(q.items))] = c
|
||||
}
|
||||
q.mu.Unlock()
|
||||
select {
|
||||
case q.notify <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (q *sampleQueue) len() int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return len(q.items)
|
||||
}
|
||||
|
||||
func (q *sampleQueue) pop() (model.Candidate, bool) {
|
||||
q.mu.Lock()
|
||||
n := len(q.items)
|
||||
if n == 0 {
|
||||
q.mu.Unlock()
|
||||
return model.Candidate{}, false
|
||||
}
|
||||
i := mathrand.IntN(n)
|
||||
c := q.items[i]
|
||||
q.items[i] = q.items[n-1]
|
||||
q.items = q.items[:n-1]
|
||||
q.mu.Unlock()
|
||||
return c, true
|
||||
}
|
||||
|
||||
func New(cfg Config) *Prober {
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 4
|
||||
@@ -142,7 +93,7 @@ func New(cfg Config) *Prober {
|
||||
}
|
||||
return &Prober{
|
||||
cfg: cfg,
|
||||
sample: newSampleQueue(cfg.Queue),
|
||||
sample: sample.New[model.Candidate](cfg.Queue),
|
||||
stop: make(chan struct{}),
|
||||
client: &http.Client{
|
||||
Timeout: cfg.Timeout,
|
||||
@@ -178,7 +129,7 @@ func (p *Prober) Submit(c model.Candidate) {
|
||||
if p.cfg.Metrics != nil {
|
||||
p.cfg.Metrics.ProbeAttempts.Add(1)
|
||||
}
|
||||
p.sample.push(c)
|
||||
p.sample.Push(c)
|
||||
}
|
||||
|
||||
// Close stops the workers after any in-flight probe. It must be called before
|
||||
@@ -188,7 +139,7 @@ func (p *Prober) Close() { close(p.stop) }
|
||||
func (p *Prober) Wait() { p.wg.Wait() }
|
||||
|
||||
// QueueLen reports the current probe sample depth.
|
||||
func (p *Prober) QueueLen() int { return p.sample.len() }
|
||||
func (p *Prober) QueueLen() int { return p.sample.Len() }
|
||||
|
||||
func (p *Prober) worker(ctx context.Context) {
|
||||
defer p.wg.Done()
|
||||
@@ -198,10 +149,10 @@ func (p *Prober) worker(ctx context.Context) {
|
||||
return
|
||||
case <-p.stop:
|
||||
return
|
||||
case <-p.sample.notify:
|
||||
case <-p.sample.Notify():
|
||||
}
|
||||
for {
|
||||
c, ok := p.sample.pop()
|
||||
c, ok := p.sample.Pop()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
@@ -225,12 +176,6 @@ func (p *Prober) worker(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Probe is the (exposure, detail) convenience view.
|
||||
func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
r := p.ProbeDetailed(ctx, c)
|
||||
return r.Exposure, r.Detail
|
||||
}
|
||||
|
||||
// ProbeDetailed runs the vendor-appropriate check and reports capabilities.
|
||||
func (p *Prober) ProbeDetailed(ctx context.Context, c model.Candidate) Result {
|
||||
// Deep, read-only protocol checks take precedence for database services.
|
||||
@@ -469,8 +414,3 @@ func classifyError(err error) (model.ExposureState, string) {
|
||||
}
|
||||
return model.ExposureUnknown, err.Error()
|
||||
}
|
||||
|
||||
// Describe is a helper for logs/tests.
|
||||
func Describe(s model.ExposureState) string {
|
||||
return fmt.Sprintf("exposure=%s", s)
|
||||
}
|
||||
@@ -38,11 +38,11 @@ func TestProbeStorageStatuses(t *testing.T) {
|
||||
for _, tc := range cases {
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, statusHandler(tc.status))
|
||||
got, detail := p.Probe(context.Background(), model.Candidate{
|
||||
r := p.ProbeDetailed(context.Background(), model.Candidate{
|
||||
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
||||
})
|
||||
if got != tc.want {
|
||||
t.Errorf("status %d: got %s (%s), want %s", tc.status, got, detail, tc.want)
|
||||
if r.Exposure != tc.want {
|
||||
t.Errorf("status %d: got %s (%s), want %s", tc.status, r.Exposure, r.Detail, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -54,29 +54,29 @@ func TestProbeS3RegionRedirect(t *testing.T) {
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, h)
|
||||
got, detail := p.Probe(context.Background(), model.Candidate{
|
||||
r := p.ProbeDetailed(context.Background(), model.Candidate{
|
||||
Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
||||
})
|
||||
if got != model.ExposureReachable || !strings.Contains(detail, "eu-central-1") {
|
||||
t.Fatalf("got %s (%s), want reachable with region", got, detail)
|
||||
if r.Exposure != model.ExposureReachable || !strings.Contains(r.Detail, "eu-central-1") {
|
||||
t.Fatalf("got %s (%s), want reachable with region", r.Exposure, r.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeFirebase(t *testing.T) {
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, statusHandler(http.StatusUnauthorized))
|
||||
got, _ := p.Probe(context.Background(), model.Candidate{
|
||||
r := p.ProbeDetailed(context.Background(), model.Candidate{
|
||||
Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
|
||||
})
|
||||
if got != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", got)
|
||||
if r.Exposure != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", r.Exposure)
|
||||
}
|
||||
host2 := serverHost(t, statusHandler(http.StatusOK))
|
||||
got, _ = p.Probe(context.Background(), model.Candidate{
|
||||
r = p.ProbeDetailed(context.Background(), model.Candidate{
|
||||
Hostname: host2, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB,
|
||||
})
|
||||
if got != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s, want public_read", got)
|
||||
if r.Exposure != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s, want public_read", r.Exposure)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,11 +90,11 @@ func TestProbeAzureContainerDiscovery(t *testing.T) {
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
host := serverHost(t, h)
|
||||
state, detail := p.Probe(context.Background(), model.Candidate{
|
||||
r := p.ProbeDetailed(context.Background(), model.Candidate{
|
||||
Hostname: host, Resource: "acct", Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
|
||||
})
|
||||
if state != model.ExposurePublicRead || !strings.Contains(detail, "images") {
|
||||
t.Fatalf("got %s (%s), want public_read via container 'images'", state, detail)
|
||||
if r.Exposure != model.ExposurePublicRead || !strings.Contains(r.Detail, "images") {
|
||||
t.Fatalf("got %s (%s), want public_read via container 'images'", r.Exposure, r.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user