over-engineering audit: remove dead code, speculative config, duplicate queues
- delete embedded community resolver snapshot (6,760 lines) + EmbeddedServers + DNS_SERVERS=fresh; curated resolvers are the default. Simplify Warm (no sampling). - one generic internal/sample.Queue[T] replaces three near-identical random-sample queues (probe, pipeline, discover). - delete unreachable index.listOpenSearch (OpenSearch is never listable). - delete classifyCloudFront (its match is always discarded, !ResourceSpecific). - delete probe.Probe wrapper + Describe; tests use ProbeDetailed. - delete store.Pool(), model.SourceDNSAlias, discover.QueueLen. - drop dead config/no-op knobs: pipeline DirectWorkers, discover MaxPerDomain, discover S3/GCS/DO flags, BUCKET_SCAN_DO. ~-7,000 lines (mostly the resolver snapshot), 0 deps removed, tests green.
This commit is contained in:
StasonJatham committed
1 parent
81e53f8755
commit
dc7bdae99d
22 files changed
+150
-7202
No files matched your search
@@ -12,7 +12,6 @@ package discover
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -22,18 +21,13 @@ import (
|
||||
"tea.karlcom.de/karl/bucketeer/internal/index"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/metrics"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/sample"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
QPS int // global outbound request rate
|
||||
Workers int // concurrent candidate probes
|
||||
Queue int // sampled domain backlog
|
||||
S3 bool
|
||||
GCS bool
|
||||
DOSpaces bool
|
||||
DORegions []string
|
||||
MaxPerDomain int // cap candidates generated per domain
|
||||
|
||||
QPS int // global outbound request rate
|
||||
Workers int // concurrent candidate probes
|
||||
Queue int // sampled domain backlog
|
||||
Indexer *index.Indexer
|
||||
Metrics *metrics.Metrics
|
||||
// OnOpen is called for every anonymously listable bucket with its metadata
|
||||
@@ -44,7 +38,7 @@ type Config struct {
|
||||
type Discoverer struct {
|
||||
cfg Config
|
||||
client *http.Client
|
||||
q *domainQueue
|
||||
q *sample.Queue[string]
|
||||
tokens chan struct{}
|
||||
seen *dns.TTLCache[bool] // candidate names already tested
|
||||
stop chan struct{}
|
||||
@@ -61,13 +55,10 @@ func New(cfg Config) *Discoverer {
|
||||
if cfg.Queue <= 0 {
|
||||
cfg.Queue = 65536
|
||||
}
|
||||
if cfg.MaxPerDomain <= 0 {
|
||||
cfg.MaxPerDomain = 48
|
||||
}
|
||||
return &Discoverer{
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: 6 * time.Second},
|
||||
q: newDomainQueue(cfg.Queue),
|
||||
q: sample.New[string](cfg.Queue),
|
||||
tokens: make(chan struct{}, max(cfg.QPS, 1)),
|
||||
seen: dns.NewTTLCache[bool](1<<20, 64),
|
||||
stop: make(chan struct{}),
|
||||
@@ -77,7 +68,7 @@ func New(cfg Config) *Discoverer {
|
||||
// Submit offers a hostname to the discovery engine. Duplicate registrable
|
||||
// domains and already-tested candidate names are dropped cheaply.
|
||||
func (d *Discoverer) Submit(host string) {
|
||||
d.q.push(host)
|
||||
d.q.Push(host)
|
||||
}
|
||||
|
||||
func (d *Discoverer) Start(ctx context.Context) {
|
||||
@@ -92,7 +83,6 @@ func (d *Discoverer) Close() { close(d.stop) }
|
||||
func (d *Discoverer) Wait() { d.wg.Wait() }
|
||||
|
||||
// QueueLen reports the sampled domain backlog.
|
||||
func (d *Discoverer) QueueLen() int { return d.q.len() }
|
||||
|
||||
func (d *Discoverer) refill(ctx context.Context) {
|
||||
period := time.Second / time.Duration(d.cfg.QPS)
|
||||
@@ -135,10 +125,10 @@ func (d *Discoverer) worker(ctx context.Context) {
|
||||
return
|
||||
case <-d.stop:
|
||||
return
|
||||
case <-d.q.notify:
|
||||
case <-d.q.Notify():
|
||||
}
|
||||
for {
|
||||
host, ok := d.q.pop()
|
||||
host, ok := d.q.Pop()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
@@ -148,7 +138,7 @@ func (d *Discoverer) worker(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (d *Discoverer) scanDomain(ctx context.Context, host string) {
|
||||
for _, name := range candidates(host, d.cfg.MaxPerDomain) {
|
||||
for _, name := range candidates(host) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
@@ -183,35 +173,21 @@ type Target struct {
|
||||
}
|
||||
|
||||
func (d *Discoverer) targets(name string) []Target {
|
||||
var out []Target
|
||||
if d.cfg.S3 {
|
||||
out = append(out, Target{
|
||||
return []Target{
|
||||
{
|
||||
Name: name, Provider: model.ProviderAWS, Service: model.ServiceS3,
|
||||
Hostname: name + ".s3.amazonaws.com",
|
||||
URL: "https://" + name + ".s3.amazonaws.com/?list-type=2&max-keys=1",
|
||||
})
|
||||
}
|
||||
if d.cfg.GCS {
|
||||
// Use the virtual-host form so each bucket has a unique hostname (the
|
||||
// path form makes every bucket share storage.googleapis.com and collide
|
||||
// on the cloud_assets unique key).
|
||||
out = append(out, Target{
|
||||
},
|
||||
{
|
||||
// Virtual-host form so each bucket has a unique hostname (the path
|
||||
// form makes every bucket share storage.googleapis.com and collide
|
||||
// on the cloud_assets unique key).
|
||||
Name: name, Provider: model.ProviderGCP, Service: model.ServiceObjectStorage,
|
||||
Hostname: name + ".storage.googleapis.com",
|
||||
URL: "https://storage.googleapis.com/" + name + "?max-keys=1",
|
||||
})
|
||||
},
|
||||
}
|
||||
if d.cfg.DOSpaces {
|
||||
for _, region := range d.cfg.DORegions {
|
||||
host := name + "." + region + ".digitaloceanspaces.com"
|
||||
out = append(out, Target{
|
||||
Name: name, Provider: model.ProviderDigitalOcean, Service: model.ServiceObjectStorage,
|
||||
Hostname: host, Region: region,
|
||||
URL: "https://" + host + "/?list-type=2&max-keys=1",
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (d *Discoverer) test(ctx context.Context, t Target) {
|
||||
@@ -286,13 +262,7 @@ var skipLabel = map[string]bool{
|
||||
"ns": true, "mx": true, "localhost": true,
|
||||
}
|
||||
|
||||
func candidates(host string, max int) []string {
|
||||
out := baseLabels(host)
|
||||
if max > 0 && len(out) > max {
|
||||
out = out[:max]
|
||||
}
|
||||
return out
|
||||
}
|
||||
func candidates(host string) []string { return baseLabels(host) }
|
||||
|
||||
func validLabel(s string) bool {
|
||||
if len(s) < 3 || len(s) > 40 {
|
||||
@@ -319,51 +289,3 @@ func validBucketName(s string) bool {
|
||||
}
|
||||
return s[0] != '-' && s[0] != '.' && s[len(s)-1] != '-' && s[len(s)-1] != '.'
|
||||
}
|
||||
|
||||
// --- sampled domain queue ---
|
||||
|
||||
type domainQueue struct {
|
||||
mu sync.Mutex
|
||||
items []string
|
||||
cap int
|
||||
notify chan struct{}
|
||||
}
|
||||
|
||||
func newDomainQueue(capacity int) *domainQueue {
|
||||
return &domainQueue{cap: capacity, notify: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
func (q *domainQueue) push(s string) {
|
||||
q.mu.Lock()
|
||||
if len(q.items) < q.cap {
|
||||
q.items = append(q.items, s)
|
||||
} else {
|
||||
q.items[rand.IntN(len(q.items))] = s
|
||||
}
|
||||
q.mu.Unlock()
|
||||
select {
|
||||
case q.notify <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (q *domainQueue) pop() (string, bool) {
|
||||
q.mu.Lock()
|
||||
n := len(q.items)
|
||||
if n == 0 {
|
||||
q.mu.Unlock()
|
||||
return "", false
|
||||
}
|
||||
i := rand.IntN(n)
|
||||
s := q.items[i]
|
||||
q.items[i] = q.items[n-1]
|
||||
q.items = q.items[:n-1]
|
||||
q.mu.Unlock()
|
||||
return s, true
|
||||
}
|
||||
|
||||
func (q *domainQueue) len() int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return len(q.items)
|
||||
}
|
||||
Reference in new issue
Block a user