Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs
Logging: - structured slog with LOG_FORMAT=json|text and a stable key vocabulary - periodic stats record with queue depths, db pool, goroutines, memory, dns/probe - no per-certificate logging (hot path is counters only) UI (Bucketeer branding, DESIGN.md): - light+dark themes (OS default, persisted header toggle, no first-paint flash) - official mascot: lossless WebP light/dark logos + favicon, embedded and served with cache headers (1.2MB source PNGs -> ~27/22KB webp) - Assets/Contents views, pagination (offset) and native virtual scrolling (content-visibility) so large result sets do not blow up the DOM - search with debounce/abort + URL state, exposure chips, sort, summary counts Readability + indexing: - internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch (metadata only: keys/sizes/dates; never object contents) - asset_entries table + store write/search; readable resources are indexed and searchable in the UI - deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING, PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune HTTP reads - Azure account-level listing fixed: 403 is reported reachable, not auth_required, since anonymous access is per-container DNS: - curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at once broke the container resolver) - random healthy selection + failover + per-server cooldown Other: prober Close fixes a shutdown hang; config/README updated.
This commit is contained in:
1 parent
8fce55f8c9
commit
a366905a43
28 files changed
+9744
-241
No files matched your search
@@ -31,15 +31,19 @@ type Config struct {
|
||||
DBBatchSize int
|
||||
DBFlushInterval time.Duration
|
||||
|
||||
ActiveProbing bool
|
||||
ProbeWorkers int
|
||||
ProbeTimeout time.Duration
|
||||
ProbeQueueSize int
|
||||
ActiveProbing bool
|
||||
ProbeWorkers int
|
||||
ProbeTimeout time.Duration
|
||||
ProbeQueueSize int
|
||||
ProbeDeep bool
|
||||
IndexEnabled bool
|
||||
IndexMaxObjects int
|
||||
|
||||
MetricsAddr string
|
||||
UIUser string
|
||||
UIPass string
|
||||
LogLevel string
|
||||
LogFormat string
|
||||
StatsInterval time.Duration
|
||||
|
||||
LogListSource string
|
||||
@@ -51,15 +55,11 @@ type Config struct {
|
||||
// DefaultLogListSource is the public CT log list used by certspotter.
|
||||
const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.json"
|
||||
|
||||
// DefaultDNSServers are load-balanced across. Set DNS_SERVERS=system to use the
|
||||
// host resolver instead, or provide your own comma-separated list.
|
||||
var DefaultDNSServers = []string{"1.1.1.1:53", "8.8.8.8:53", "1.0.0.1:53"}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
c := &Config{
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
DBMaxConns: envInt("DB_MAX_CONNS", 8),
|
||||
DNSServers: envList("DNS_SERVERS", DefaultDNSServers),
|
||||
DNSServers: envList("DNS_SERVERS", nil),
|
||||
DNSWorkers: envInt("DNS_WORKERS", 16),
|
||||
DNSTimeout: envDuration("DNS_TIMEOUT", 5*time.Second),
|
||||
DNSCacheSize: envInt("DNS_CACHE_SIZE", 1<<18),
|
||||
@@ -76,10 +76,14 @@ func Load() (*Config, error) {
|
||||
ProbeWorkers: envInt("PROBE_WORKERS", 8),
|
||||
ProbeTimeout: envDuration("PROBE_TIMEOUT", 5*time.Second),
|
||||
ProbeQueueSize: envInt("PROBE_QUEUE_SIZE", 1<<16),
|
||||
ProbeDeep: envBool("PROBE_DEEP", true),
|
||||
IndexEnabled: envBool("INDEX_ENABLED", true),
|
||||
IndexMaxObjects: envInt("INDEX_MAX_OBJECTS", 1000),
|
||||
MetricsAddr: envString("METRICS_ADDR", ":9090"),
|
||||
UIUser: envString("UI_USER", ""),
|
||||
UIPass: envString("UI_PASS", ""),
|
||||
LogLevel: envString("LOG_LEVEL", "info"),
|
||||
LogFormat: envString("LOG_FORMAT", "text"),
|
||||
StatsInterval: envDuration("STATS_INTERVAL", 60*time.Second),
|
||||
LogListSource: envString("LOG_LIST_SOURCE", DefaultLogListSource),
|
||||
StartAtEnd: envBool("START_AT_END", false),
|
||||
@@ -113,6 +117,11 @@ func (c *Config) validate() error {
|
||||
default:
|
||||
return fmt.Errorf("LOG_LEVEL must be one of debug, info, warn, error")
|
||||
}
|
||||
switch c.LogFormat {
|
||||
case "text", "json":
|
||||
default:
|
||||
return fmt.Errorf("LOG_FORMAT must be text or json")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user