bucketeer/internal/config/config.go

// Package config loads all ctcloud configuration from environment variables.
// There are no configuration files at runtime.
package config

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
)

type Config struct {
	DatabaseURL string
	DBMaxConns  int

	DNSServers       []string
	DNSWorkers       int
	DNSTimeout       time.Duration
	DNSCacheSize     int
	DNSMaxCNAMEdepth int
	DNSPositiveTTL   time.Duration
	DNSNegativeTTL   time.Duration

	DedupSize int

	DirectQueueSize int
	DNSQueueSize    int
	DBQueueSize     int

	DBBatchSize     int
	DBFlushInterval time.Duration

	ActiveProbing   bool
	ProbeWorkers    int
	ProbeTimeout    time.Duration
	ProbeQueueSize  int
	ProbeDeep       bool
	IndexEnabled    bool
	IndexMaxObjects int

	BucketScanEnabled  bool
	BucketScanQPS      int
	BucketScanWorkers  int
	BucketScanQueue    int
	BucketScanDO       bool
	BucketScanWordlist bool

	MetricsAddr   string
	UIUser        string
	UIPass        string
	LogLevel      string
	LogFormat     string
	StatsInterval time.Duration

	LogListSource   string
	StartAtEnd      bool
	ShutdownTimeout time.Duration
	DedupeTTL       time.Duration
}

// DefaultLogListSource is the public CT log list used by certspotter.
const DefaultLogListSource = "https://www.gstatic.com/ct/log_list/v3/log_list.json"

func Load() (*Config, error) {
	c := &Config{
		DatabaseURL:        os.Getenv("DATABASE_URL"),
		DBMaxConns:         envInt("DB_MAX_CONNS", 8),
		DNSServers:         envList("DNS_SERVERS", nil),
		DNSWorkers:         envInt("DNS_WORKERS", 64),
		DNSTimeout:         envDuration("DNS_TIMEOUT", 5*time.Second),
		DNSCacheSize:       envInt("DNS_CACHE_SIZE", 1<<18),
		DedupSize:          envInt("DEDUP_SIZE", 1<<18),
		DNSMaxCNAMEdepth:   envInt("DNS_MAX_CNAME_DEPTH", 8),
		DNSPositiveTTL:     envDuration("DNS_POSITIVE_TTL", 10*time.Minute),
		DNSNegativeTTL:     envDuration("DNS_NEGATIVE_TTL", 1*time.Minute),
		DirectQueueSize:    envInt("DIRECT_QUEUE_SIZE", 8192),
		DNSQueueSize:       envInt("DNS_QUEUE_SIZE", 262144),
		DBQueueSize:        envInt("DB_QUEUE_SIZE", 8192),
		DBBatchSize:        envInt("DB_BATCH_SIZE", 500),
		DBFlushInterval:    envDuration("DB_FLUSH_INTERVAL", 2*time.Second),
		ActiveProbing:      envBool("ACTIVE_PROBING", false),
		ProbeWorkers:       envInt("PROBE_WORKERS", 8),
		ProbeTimeout:       envDuration("PROBE_TIMEOUT", 5*time.Second),
		ProbeQueueSize:     envInt("PROBE_QUEUE_SIZE", 1<<16),
		ProbeDeep:          envBool("PROBE_DEEP", true),
		IndexEnabled:       envBool("INDEX_ENABLED", true),
		IndexMaxObjects:    envInt("INDEX_MAX_OBJECTS", 1000),
		BucketScanEnabled:  envBool("BUCKET_SCAN_ENABLED", true),
		BucketScanQPS:      envInt("BUCKET_SCAN_QPS", 50),
		BucketScanWorkers:  envInt("BUCKET_SCAN_WORKERS", 16),
		BucketScanQueue:    envInt("BUCKET_SCAN_QUEUE", 65536),
		BucketScanDO:       envBool("BUCKET_SCAN_DO", false),
		BucketScanWordlist: envBool("BUCKET_SCAN_WORDLIST", false),
		MetricsAddr:        envString("METRICS_ADDR", ":9090"),
		UIUser:             envString("UI_USER", ""),
		UIPass:             envString("UI_PASS", ""),
		LogLevel:           envString("LOG_LEVEL", "info"),
		LogFormat:          envString("LOG_FORMAT", "text"),
		StatsInterval:      envDuration("STATS_INTERVAL", 60*time.Second),
		LogListSource:      envString("LOG_LIST_SOURCE", DefaultLogListSource),
		StartAtEnd:         envBool("START_AT_END", false),
		ShutdownTimeout:    envDuration("SHUTDOWN_TIMEOUT", 20*time.Second),
		DedupeTTL:          envDuration("DEDUP_TTL", time.Hour),
	}
	if err := c.validate(); err != nil {
		return nil, err
	}
	return c, nil
}

func (c *Config) validate() error {
	if c.DatabaseURL == "" {
		return fmt.Errorf("DATABASE_URL is required")
	}
	if c.DNSWorkers < 1 {
		return fmt.Errorf("DNS_WORKERS must be >= 1")
	}
	if c.DNSMaxCNAMEdepth < 1 {
		return fmt.Errorf("DNS_MAX_CNAME_DEPTH must be >= 1")
	}
	if c.DirectQueueSize < 1 || c.DNSQueueSize < 1 || c.DBQueueSize < 1 {
		return fmt.Errorf("queue sizes must be >= 1")
	}
	if c.DBBatchSize < 1 {
		return fmt.Errorf("DB_BATCH_SIZE must be >= 1")
	}
	switch c.LogLevel {
	case "debug", "info", "warn", "error":
	default:
		return fmt.Errorf("LOG_LEVEL must be one of debug, info, warn, error")
	}
	switch c.LogFormat {
	case "text", "json":
	default:
		return fmt.Errorf("LOG_FORMAT must be text or json")
	}
	return nil
}

func envList(key string, def []string) []string {
	v := os.Getenv(key)
	if v == "" {
		return def
	}
	parts := strings.Split(v, ",")
	out := make([]string, 0, len(parts))
	for _, p := range parts {
		if p = strings.TrimSpace(p); p != "" {
			out = append(out, p)
		}
	}
	if len(out) == 0 {
		return def
	}
	return out
}

func envString(key, def string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return def
}

func envInt(key string, def int) int {
	if v := os.Getenv(key); v != "" {
		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
			return n
		}
	}
	return def
}

func envBool(key string, def bool) bool {
	if v := os.Getenv(key); v != "" {
		if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
			return b
		}
	}
	return def
}

func envDuration(key string, def time.Duration) time.Duration {
	if v := os.Getenv(key); v != "" {
		if d, err := time.ParseDuration(strings.TrimSpace(v)); err == nil {
			return d
		}
	}
	return def
}