Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs

Logging:
- structured slog with LOG_FORMAT=json|text and a stable key vocabulary
- periodic stats record with queue depths, db pool, goroutines, memory, dns/probe
- no per-certificate logging (hot path is counters only)

UI (Bucketeer branding, DESIGN.md):
- light+dark themes (OS default, persisted header toggle, no first-paint flash)
- official mascot: lossless WebP light/dark logos + favicon, embedded and served
  with cache headers (1.2MB source PNGs -> ~27/22KB webp)
- Assets/Contents views, pagination (offset) and native virtual scrolling
  (content-visibility) so large result sets do not blow up the DOM
- search with debounce/abort + URL state, exposure chips, sort, summary counts

Readability + indexing:
- internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch
  (metadata only: keys/sizes/dates; never object contents)
- asset_entries table + store write/search; readable resources are indexed and
  searchable in the UI
- deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING,
  PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune
  HTTP reads
- Azure account-level listing fixed: 403 is reported reachable, not
  auth_required, since anonymous access is per-container

DNS:
- curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in
  Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at
  once broke the container resolver)
- random healthy selection + failover + per-server cooldown

Other: prober Close fixes a shutdown hang; config/README updated.

internal/probe/probe.go

@@ -23,6 +23,7 @@ import (
 	"sync"
 	"time"
 
+	"tea.karlcom.de/karl/bucketeer/internal/index"
 	"tea.karlcom.de/karl/bucketeer/internal/metrics"
 	"tea.karlcom.de/karl/bucketeer/internal/model"
 )
@@ -34,14 +35,21 @@ type Config struct {
 	Workers int
 	Timeout time.Duration
 	Queue   int
+	Deep    bool // unauthenticated protocol reads for database services
 	Metrics *metrics.Metrics
 	Update  UpdateFunc
+
+	// Indexer, when set, lists metadata for resources confirmed public_read and
+	// hands the entries to OnIndex. Contents are never downloaded.
+	Indexer *index.Indexer
+	OnIndex func(ctx context.Context, c model.Candidate, entries []model.Entry)
 }
 
 type Prober struct {
 	cfg    Config
 	sample *sampleQueue
 	client *http.Client
+	stop   chan struct{}
 	wg     sync.WaitGroup
 }
 
@@ -75,6 +83,12 @@ func (q *sampleQueue) push(c model.Candidate) {
 	}
 }
 
+func (q *sampleQueue) len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return len(q.items)
+}
+
 func (q *sampleQueue) pop() (model.Candidate, bool) {
 	q.mu.Lock()
 	n := len(q.items)
@@ -103,6 +117,7 @@ func New(cfg Config) *Prober {
 	return &Prober{
 		cfg:    cfg,
 		sample: newSampleQueue(cfg.Queue),
+		stop:   make(chan struct{}),
 		client: &http.Client{
 			Timeout: cfg.Timeout,
 			// Observe the first response; do not follow redirects so region
@@ -141,14 +156,23 @@ func (p *Prober) Submit(c model.Candidate) {
 	p.sample.push(c)
 }
 
+// Close stops the workers after any in-flight probe. It must be called before
+// Wait, otherwise Wait blocks until ctx is cancelled.
+func (p *Prober) Close() { close(p.stop) }
+
 func (p *Prober) Wait() { p.wg.Wait() }
 
+// QueueLen reports the current probe sample depth.
+func (p *Prober) QueueLen() int { return p.sample.len() }
+
 func (p *Prober) worker(ctx context.Context) {
 	defer p.wg.Done()
 	for {
 		select {
 		case <-ctx.Done():
 			return
+		case <-p.stop:
+			return
 		case <-p.sample.notify:
 		}
 		for {
@@ -163,12 +187,27 @@ func (p *Prober) worker(ctx context.Context) {
 			if p.cfg.Update != nil {
 				p.cfg.Update(ctx, c, state, detail)
 			}
+			if state == model.ExposurePublicRead && p.cfg.Indexer != nil && p.cfg.OnIndex != nil {
+				if entries, err := p.cfg.Indexer.List(ctx, c); err == nil && len(entries) > 0 {
+					p.cfg.OnIndex(ctx, c, entries)
+					if p.cfg.Metrics != nil {
+						p.cfg.Metrics.EntriesIndexed.Add(uint64(len(entries)))
+					}
+				}
+			}
 		}
 	}
 }
 
 // Probe runs the vendor-appropriate check for a single candidate.
 func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
+	// Deep, read-only protocol checks take precedence for database services and
+	// can distinguish "reachable" from "actually readable without credentials".
+	if p.cfg.Deep {
+		if state, detail, ok := p.deepDB(ctx, c); ok {
+			return state, detail
+		}
+	}
 	switch c.Provider {
 	case model.ProviderAWS:
 		switch c.Service {
@@ -188,8 +227,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
 	case model.ProviderAzure:
 		switch c.Service {
 		case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
-			// Anonymous container/file/share/queue/table listing.
-			return p.checkHTTP(ctx, "https://"+c.Hostname+"/?comp=list")
+			return p.checkAzureStorage(ctx, c)
 		case model.ServiceWebApp, model.ServiceSearch:
 			return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
 		case model.ServiceCosmosDB:
@@ -230,6 +268,27 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
 	return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
 }
 
+// checkAzureStorage probes the account-level container list. Anonymous read is
+// configured per container in Azure ("Public access level"), never at the
+// account level, so a 401/403 here means only that account listing needs auth —
+// it does NOT mean the containers are unreadable. We therefore report reachable
+// rather than auth_required, and only index if account listing is unexpectedly
+// public (which would let us discover container names).
+func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
+	status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list")
+	if err != nil {
+		return classifyError(err)
+	}
+	switch {
+	case status >= 200 && status < 300:
+		return model.ExposurePublicRead, "azure account-level anonymous listing succeeded"
+	case status == http.StatusUnauthorized || status == http.StatusForbidden:
+		return model.ExposureReachable, "azure account listing requires auth; per-container readability needs a container name"
+	default:
+		return classifyStatus(status, "azure")
+	}
+}
+
 // checkS3 queries the S3 ListObjectsV2 anonymous endpoint. A 200 means the
 // bucket allows anonymous listing (public read); 403 means it exists but is not
 // public; 404 means no such bucket. 301/307 carries the real region.