Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs

Logging:
- structured slog with LOG_FORMAT=json|text and a stable key vocabulary
- periodic stats record with queue depths, db pool, goroutines, memory, dns/probe
- no per-certificate logging (hot path is counters only)

UI (Bucketeer branding, DESIGN.md):
- light+dark themes (OS default, persisted header toggle, no first-paint flash)
- official mascot: lossless WebP light/dark logos + favicon, embedded and served
  with cache headers (1.2MB source PNGs -> ~27/22KB webp)
- Assets/Contents views, pagination (offset) and native virtual scrolling
  (content-visibility) so large result sets do not blow up the DOM
- search with debounce/abort + URL state, exposure chips, sort, summary counts

Readability + indexing:
- internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch
  (metadata only: keys/sizes/dates; never object contents)
- asset_entries table + store write/search; readable resources are indexed and
  searchable in the UI
- deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING,
  PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune
  HTTP reads
- Azure account-level listing fixed: 403 is reported reachable, not
  auth_required, since anonymous access is per-container

DNS:
- curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in
  Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at
  once broke the container resolver)
- random healthy selection + failover + per-server cooldown

Other: prober Close fixes a shutdown hang; config/README updated.

cmd/ctcloud/main.go

@@ -8,8 +8,10 @@ import (
 	"errors"
 	"log/slog"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
+	"runtime"
 	"strings"
 	"sync/atomic"
 	"syscall"
@@ -19,6 +21,7 @@ import (
 
 	"tea.karlcom.de/karl/bucketeer/internal/config"
 	"tea.karlcom.de/karl/bucketeer/internal/dns"
+	"tea.karlcom.de/karl/bucketeer/internal/index"
 	"tea.karlcom.de/karl/bucketeer/internal/metrics"
 	"tea.karlcom.de/karl/bucketeer/internal/model"
 	"tea.karlcom.de/karl/bucketeer/internal/pipeline"
@@ -52,10 +55,31 @@ func run() error {
 	if err != nil {
 		return err
 	}
-	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: levelFromString(cfg.LogLevel)})))
-	if strings.Contains(cfg.DatabaseURL, "sslmode=disable") {
-		slog.Warn("DATABASE_URL has sslmode=disable; only use this on a private network")
+	opts := &slog.HandlerOptions{Level: levelFromString(cfg.LogLevel)}
+	var handler slog.Handler = slog.NewTextHandler(os.Stderr, opts)
+	if cfg.LogFormat == "json" {
+		handler = slog.NewJSONHandler(os.Stderr, opts) // machine/LLM friendly
 	}
+	slog.SetDefault(slog.New(handler))
+	if strings.Contains(cfg.DatabaseURL, "sslmode=disable") {
+		slog.Warn("insecure database transport", "event", "config_warning",
+			"detail", "DATABASE_URL has sslmode=disable; only use this on a private network")
+	}
+	slog.Info("configuration",
+		"event", "startup",
+		"database_host", redactURLHost(cfg.DatabaseURL),
+		"metrics_addr", cfg.MetricsAddr,
+		"ui_auth", cfg.UIUser != "",
+		"active_probing", cfg.ActiveProbing,
+		"log_list_source", cfg.LogListSource,
+		"start_at_end", cfg.StartAtEnd,
+		"dns_workers", cfg.DNSWorkers,
+		"direct_queue", cfg.DirectQueueSize,
+		"dns_queue", cfg.DNSQueueSize,
+		"db_queue", cfg.DBQueueSize,
+		"db_batch_size", cfg.DBBatchSize,
+		"db_flush_interval", cfg.DBFlushInterval.String(),
+	)
 
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
@@ -73,8 +97,29 @@ func run() error {
 	}
 	slog.Info("database ready")
 
+	dnsServers := cfg.DNSServers
+	var resolver *dns.ServerPool
+	switch {
+	case len(dnsServers) == 0:
+		// Default: a curated set of reliable public resolvers. Deliberately not
+		// the community list by default: warming thousands of unknown resolvers
+		// at once has been observed to saturate and break the container's own
+		// resolver, taking down the CT log-list fetch.
+		resolver = dns.NewServerPool(dns.CuratedServers())
+		slog.Info("dns resolvers configured", "count", resolver.Len(), "set", "curated")
+	case len(dnsServers) == 1 && dnsServers[0] == "fresh":
+		resolver = dns.NewServerPool(append(dns.CuratedServers(), dns.EmbeddedServers()...))
+		warmCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		healthy := resolver.Warm(warmCtx, "example.com", 32, 256)
+		cancel()
+		slog.Info("dns resolvers warmed", "total", resolver.Len(), "healthy", healthy, "set", "fresh")
+	default:
+		resolver = dns.NewServerPool(dnsServers)
+		slog.Info("dns resolvers configured", "count", resolver.Len(), "set", "custom")
+	}
+
 	enricher := &dns.Enricher{
-		Resolver:    dns.NewServerPool(cfg.DNSServers),
+		Resolver:    resolver,
 		Cache:       dns.NewTTLCache[dns.Result](cfg.DNSCacheSize, 64),
 		PositiveTTL: cfg.DNSPositiveTTL,
 		NegativeTTL: cfg.DNSNegativeTTL,
@@ -85,17 +130,30 @@ func run() error {
 
 	var prober *probe.Prober
 	if cfg.ActiveProbing {
-		prober = probe.New(probe.Config{
+		pcfg := probe.Config{
 			Workers: cfg.ProbeWorkers,
 			Timeout: cfg.ProbeTimeout,
 			Queue:   cfg.ProbeQueueSize,
+			Deep:    cfg.ProbeDeep,
 			Metrics: m,
 			Update: func(uctx context.Context, c model.Candidate, state model.ExposureState, detail string) {
 				_ = st.UpdateExposure(uctx, c.Provider.String(), c.Service.String(), c.Hostname, state, detail)
 			},
-		})
+		}
+		if cfg.IndexEnabled {
+			pcfg.Indexer = index.New(cfg.IndexMaxObjects, 2*cfg.ProbeTimeout)
+			pcfg.OnIndex = func(uctx context.Context, c model.Candidate, entries []model.Entry) {
+				if n, err := st.ReplaceEntries(uctx, c.Provider.String(), c.Service.String(), c.Hostname, entries); err != nil {
+					slog.Warn("index write failed", "event", "index_error", "host", c.Hostname, "error", err)
+				} else if n > 0 {
+					slog.Info("indexed readable resource", "event", "indexed",
+						"host", c.Hostname, "provider", c.Provider.String(), "service", c.Service.String(), "entries", n)
+				}
+			}
+		}
+		prober = probe.New(pcfg)
 		prober.Start(ctx)
-		slog.Info("active probing enabled")
+		slog.Info("active probing enabled", "event", "startup", "indexing", cfg.IndexEnabled, "index_max_objects", cfg.IndexMaxObjects)
 	}
 
 	write := func(wctx context.Context, batch []pipeline.Finding) error {
@@ -141,7 +199,7 @@ func run() error {
 
 	ui := webui.New(st, webui.Config{User: cfg.UIUser, Pass: cfg.UIPass})
 	httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady)
-	go logStats(ctx, m, cfg.StatsInterval)
+	go logStats(ctx, m, p, st, resolver, prober, cfg.StatsInterval)
 
 	slog.Info("ctcloud started")
 	err = <-monitorDone
@@ -158,6 +216,7 @@ func run() error {
 	go func() {
 		p.Close()
 		if prober != nil {
+			prober.Close()
 			prober.Wait()
 		}
 		close(drained)
@@ -213,7 +272,10 @@ func startHTTP(addr string, m *metrics.Metrics, st *store.Store, ui http.Handler
 	return srv
 }
 
-func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) {
+// logStats emits one structured record per interval with a stable key
+// vocabulary, so it can be parsed by an LLM or a log pipeline. It never logs per
+// certificate; per-certificate hot-path events are only counted.
+func logStats(ctx context.Context, m *metrics.Metrics, p *pipeline.Pipeline, st *store.Store, resolver *dns.ServerPool, prober *probe.Prober, interval time.Duration) {
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 	for {
@@ -221,7 +283,12 @@ func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			slog.Info("stats",
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+			direct, dnsQ, dbQ := p.Depths()
+			dbTotal, dbIdle, dbAcquired := st.PoolStats()
+			attrs := []any{
+				"event", "stats",
 				"certificates", m.CTCertificates.Load(),
 				"identifiers", m.CTIdentifiers.Load(),
 				"direct_matches", m.DirectMatches.Load(),
@@ -230,10 +297,39 @@ func logStats(ctx context.Context, m *metrics.Metrics, interval time.Duration) {
 				"dns_queries", m.DNSQueries.Load(),
 				"dns_cache_hits", m.DNSCacheHits.Load(),
 				"dns_errors", m.DNSErrors.Load(),
+				"dns_resolvers", resolver.Len(),
+				"ct_errors", m.CTErrors.Load(),
+				"ct_malformed", m.MalformedCT.Load(),
 				"db_rows", m.DBRows.Load(),
 				"db_errors", m.DBErrors.Load(),
+				"db_pool_total", dbTotal,
+				"db_pool_idle", dbIdle,
+				"db_pool_acquired", dbAcquired,
+				"queue_direct", direct,
+				"queue_dns", dnsQ,
+				"queue_db", dbQ,
 				"queue_drops", m.QueueDrops.Load(),
-			)
+				"goroutines", runtime.NumGoroutine(),
+				"mem_sys_bytes", ms.Sys,
+				"mem_heap_bytes", ms.HeapInuse,
+				"gc_cycles", ms.NumGC,
+			}
+			if prober != nil {
+				attrs = append(attrs,
+					"probe_queue", prober.QueueLen(),
+					"probe_attempts", m.ProbeAttempts.Load(),
+					"probe_public", m.ProbePublic.Load(),
+				)
+			}
+			slog.Info("periodic stats", attrs...)
 		}
 	}
 }
+
+func redactURLHost(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil || u.Host == "" {
+		return "unparseable"
+	}
+	return u.Host
+}