professionalize repo for public release

- rename ctcloud -> bucketeer everywhere (cmd/bucketeer, binary, compose service)
- group brand assets under assets/ (light/dark logo + UI screenshot)
- move long-form docs under docs/ (ARCHITECTURE, DESIGN, GOAL) and add
  docs/SKILL.md: how an agent queries/audits a running instance and answers
  user questions from the API and PostgreSQL
- README: large logo (light/dark picture) and UI screenshot, repository layout
- .gitignore: /bucketeer, .env, pgdata, build/test artifacts, OS junk
- .dockerignore: keep brand/docs/markdown out of the build context

.dockerignore

@@ -1,6 +1,10 @@
 .git
 .gitignore
+.env
 pgdata
 *.test
 *.out
 .gocache
+assets
+docs
+*.md

.env.example

@@ -1,14 +1,14 @@
 # Copy to .env and edit. .env is gitignored.
 
-POSTGRES_USER=ctcloud
+POSTGRES_USER=bucketeer
 POSTGRES_PASSWORD=change-me-strong
-POSTGRES_DB=ctcloud
+POSTGRES_DB=bucketeer
 
 # Local compose (private bridge network, no TLS):
-DATABASE_URL=postgres://ctcloud:change-me-strong@postgres:5432/ctcloud?sslmode=disable
+DATABASE_URL=postgres://bucketeer:change-me-strong@postgres:5432/bucketeer?sslmode=disable
 
 # Production: use a TLS-enabled database instead, e.g.
-# DATABASE_URL=postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full
+# DATABASE_URL=postgres://user:[email protected]:5432/bucketeer?sslmode=verify-full
 
 # DNS_SERVERS unset = ~20 curated public resolvers. Options:
 #   DNS_SERVERS=system             use the OS resolver

.gitignore

@@ -1,5 +1,14 @@
-/ctcloud
+# Build output
+/bucketeer
+/dist/
 *.test
 *.out
-pgdata/
+coverage.*
+
+# Local configuration and data
 .env
+pgdata/
+
+# OS / editor
+.DS_Store
+*.swp

Dockerfile

@@ -13,10 +13,10 @@ COPY . .
 # Static binary: no cgo, no shell, no writable filesystem required.
 RUN CGO_ENABLED=0 GOOS=linux go build \
         -trimpath -ldflags="-s -w" \
-        -o /out/ctcloud ./cmd/ctcloud
+        -o /out/bucketeer ./cmd/bucketeer
 
 FROM gcr.io/distroless/static-debian12:nonroot
-COPY --from=build /out/ctcloud /ctcloud
+COPY --from=build /out/bucketeer /bucketeer
 USER nonroot:nonroot
 EXPOSE 9090
-ENTRYPOINT ["/ctcloud"]
+ENTRYPOINT ["/bucketeer"]

README.md

@@ -1,9 +1,18 @@
-# ctcloud
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="assets/bucketeer-dark.png">
+    <img src="assets/bucketeer-light.png" alt="Bucketeer" width="220">
+  </picture>
+</p>
 
-`ctcloud` continuously consumes Certificate Transparency (CT) logs, finds
+# Bucketeer
+
+Bucketeer continuously consumes Certificate Transparency (CT) logs, finds
 cloud storage/database/data-service resources referenced by certificates,
-enriches candidates through DNS/CNAME resolution, and persists only useful
-findings to PostgreSQL.
+enriches candidates through DNS/CNAME resolution, discovers and indexes open
+buckets, and persists only useful findings to PostgreSQL.
+
+![Bucketeer UI](assets/bucketeer-landing.jpg)
 
 It is a **single stateless Go process plus PostgreSQL**. CertSpotter's real CT
 fetching, validation and parsing code is embedded and called **in-process**;
@@ -11,7 +20,7 @@ there is no stdout handoff, no JSON handoff, no hooks, no subprocesses and no
 certificate files.
 
 > A detected provider endpoint **does not** mean the underlying data is publicly
-> accessible. `ctcloud` reports facts (hostname existed in CT, resolved, answered
+> accessible. `bucketeer` reports facts (hostname existed in CT, resolved, answered
 > a request) separately from conclusions (`exposure_state`). Most findings are
 > simply an endpoint that exists.
 
@@ -38,7 +47,35 @@ CT logs ──► CertSpotter internals (fetch/verify/parse)
            PostgreSQL
 ```
 
-See [ARCHITECTURE.md](ARCHITECTURE.md) for per-stage rules.
+See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for per-stage rules.
+
+## Repository layout
+
+```text
+cmd/bucketeer/            application entry point
+internal/
+  classify/               zero-allocation provider/resource classifier
+  config/                 environment configuration
+  discover/               bucket-name discovery from CT domains
+  dns/                    resolver pool, TTL cache, CNAME following
+  index/                  anonymous listing -> metadata index
+  metrics/                Prometheus text endpoint
+  model/                  shared enums and structs
+  pipeline/               bounded queues, dedup, worker pools
+  probe/                  reachability + readability checks
+  sample/                 bounded random-sample queue
+  scan/                   on-demand scan and backfill
+  store/                  PostgreSQL schema, batch writer, CT state
+  webui/                  embedded web UI
+migrations/               embedded SQL schema
+third_party/certspotter/  vendored upstream CT engine (minimal patch)
+docs/                     ARCHITECTURE.md, DESIGN.md, GOAL.md, SKILL.md
+assets/                   brand assets
+```
+
+More: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) (data flow),
+[docs/DESIGN.md](docs/DESIGN.md) (UI design system),
+[docs/SKILL.md](docs/SKILL.md) (how an agent queries and audits a running instance).
 
 ## Why CertSpotter is embedded (and patched)
 
@@ -48,7 +85,7 @@ tree maintenance, entry parsing and certificate reconstruction.
 
 The only change is a two-hunk patch in `third_party/certspotter` that adds an
 `IdentifierSink` callback. After a certificate is parsed, the identifiers are
-handed to ctcloud with a direct Go function call and the watchlist/chain/
+handed to bucketeer with a direct Go function call and the watchlist/chain/
 notification machinery is skipped:
 
 ```go
@@ -147,7 +184,7 @@ cp .env.example .env      # set a strong POSTGRES_PASSWORD and matching DATABASE
 docker compose up --build
 ```
 
-That starts `postgres` and the stateless `ctcloud` container. The application
+That starts `postgres` and the stateless `bucketeer` container. The application
 container runs as non-root, has a read-only root filesystem, drops all Linux
 capabilities, sets CPU/memory/pids limits and mounts no application volume. The
 only persistent state is PostgreSQL (CT checkpoints + cloud assets + index).
@@ -169,7 +206,7 @@ basic auth on the UI and its API.
 ## Web UI
 
 A small, dependency-free UI (vanilla JS, no build step, embedded in the binary)
-styled to the tokens and layout in `DESIGN.md` (neutral canvas, single 768px
+styled to the tokens and layout in [docs/DESIGN.md](docs/DESIGN.md) (neutral canvas, single 768px
 results column, flat rows, pill controls, no shadows).
 
 * search box with 250ms debounce, stale-request cancellation and `?q=` URL state
@@ -201,8 +238,8 @@ stay crisp on HiDPI displays while shrinking the ~1.2 MB source PNGs by ~96%.
 Regenerate after replacing the source PNGs at the repo root:
 
 ```sh
-magick bucketeer-light.png -resize 256x -strip /tmp/l.png
-magick bucketeer-dark.png  -resize 256x -strip /tmp/d.png
+magick assets/bucketeer-light.png -resize 256x -strip /tmp/l.png
+magick assets/bucketeer-dark.png  -resize 256x -strip /tmp/d.png
 cwebp -quiet -lossless -z 9 -alpha_q 100 /tmp/l.png -o internal/webui/assets/logo-light.webp
 cwebp -quiet -lossless -z 9 -alpha_q 100 /tmp/d.png -o internal/webui/assets/logo-dark.webp
 ```
@@ -249,7 +286,7 @@ attempted`). Database endpoints are never claimed to be readable: a raw
 * **Database TLS** — the local compose stack uses `sslmode=disable` because the
   compose bridge network is private. For production, point `DATABASE_URL` at a
   TLS-enabled database, e.g.
-  `postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full`.
+  `postgres://user:[email protected]:5432/bucketeer?sslmode=verify-full`.
   pgx handles TLS natively; no extra flags are needed.
 * **Metrics** — bound to loopback. There is no auth, no debug endpoints and no
   high-cardinality labels (never hostnames).
@@ -367,7 +404,7 @@ go vet ./...
 Integration tests for PostgreSQL run only when `DATABASE_URL` is set:
 
 ```sh
-DATABASE_URL=postgres://ctcloud:ctcloud@localhost:5432/ctcloud go test ./internal/store/
+DATABASE_URL=postgres://bucketeer:bucketeer@localhost:5432/bucketeer go test ./internal/store/
 ```
 
 ## Limitations

cmd/bucketeer/main.go

@@ -1,4 +1,4 @@
-// Command ctcloud continuously consumes Certificate Transparency logs through
+// Command bucketeer continuously consumes Certificate Transparency logs through
 // embedded CertSpotter internals, classifies cloud/database/storage resources
 // in-process and persists only useful, enriched findings to PostgreSQL.
 package main
@@ -242,7 +242,7 @@ func run() error {
 	httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady)
 	go logStats(ctx, m, p, st, resolver, prober, cfg.StatsInterval)
 
-	slog.Info("ctcloud started")
+	slog.Info("bucketeer started")
 	err = <-monitorDone
 	if err != nil && !errors.Is(err, context.Canceled) {
 		slog.Error("ct monitor stopped", "error", err)

docker-compose.yml

@@ -3,7 +3,7 @@
 # Secrets come from .env (gitignored). Copy .env.example to .env and set a
 # strong POSTGRES_PASSWORD plus a matching DATABASE_URL before `docker compose
 # up`. For a production database, point DATABASE_URL at a TLS-enabled endpoint,
-# e.g. postgres://user:[email protected]:5432/ctcloud?sslmode=verify-full
+# e.g. postgres://user:[email protected]:5432/bucketeer?sslmode=verify-full
 #
 # The compose bridge network is private; sslmode=disable is only acceptable here
 # because the database is not reachable from outside the stack.
@@ -11,13 +11,13 @@ services:
   postgres:
     image: postgres:17-alpine
     environment:
-      POSTGRES_USER: ${POSTGRES_USER:-ctcloud}
+      POSTGRES_USER: ${POSTGRES_USER:-bucketeer}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
-      POSTGRES_DB: ${POSTGRES_DB:-ctcloud}
+      POSTGRES_DB: ${POSTGRES_DB:-bucketeer}
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-ctcloud} -d ${POSTGRES_DB:-ctcloud}"]
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-bucketeer} -d ${POSTGRES_DB:-bucketeer}"]
       interval: 5s
       timeout: 5s
       retries: 12
@@ -29,13 +29,13 @@ services:
       options: { max-size: "10m", max-file: "3" }
     restart: unless-stopped
 
-  ctcloud:
+  bucketeer:
     build: .
     depends_on:
       postgres:
         condition: service_healthy
-    # All optional tuning (DNS_SERVERS, PROBE_*, UI_USER, ...) comes from .env.
-    # Leave DNS_SERVERS unset to use the curated + community resolver list.
+    # All optional tuning (DNS_SERVERS, PROBE_*, BUCKET_SCAN_*, UI_USER, ...)
+    # comes from .env. Leave DNS_SERVERS unset to use the curated resolvers.
     env_file:
       - .env
     environment:

docs/ARCHITECTURE.md

@@ -66,7 +66,7 @@ Certificate Transparency Logs
 ## Failure handling
 
 * DNS and CT network errors are retried by CertSpotter with bounded exponential
-  backoff; ctcloud counts them (`ct_errors_total`) without logging per event.
+  backoff; bucketeer counts them (`ct_errors_total`) without logging per event.
 * PostgreSQL writes retry a few times with backoff; persistent failure drops
   items from the bounded DB queue rather than growing memory.
 * The process uses `context` throughout and on `SIGTERM` stops intake, drains

docs/GOAL.md

@@ -1157,7 +1157,7 @@ docker-compose.yml
 Docker Compose should contain:
 
 ```text
-ctcloud
+bucketeer
 postgres
 ```
 
@@ -1172,7 +1172,7 @@ Keep the project simple.
 Suggested layout:
 
 ```text
-cmd/ctcloud/
+cmd/bucketeer/
     main.go
 
 internal/

docs/SKILL.md

@@ -0,0 +1,110 @@
+---
+name: bucketeer-ops
+description: Query and audit a running Bucketeer instance (CT cloud-resource monitor) over its HTTP API and PostgreSQL, and answer user questions about discovered assets, open buckets, indexed objects and databases.
+---
+
+# Operating and auditing Bucketeer
+
+Bucketeer is a single Go process + PostgreSQL that consumes CT logs, classifies
+cloud resources, discovers open buckets, and indexes their contents. This skill is
+for an agent that needs to **answer questions from the data** without reading the
+source.
+
+## Access
+
+Assume a local deployment (`docker compose`):
+
+- API base: `http://127.0.0.1:9090` (override with `METRICS_ADDR` / `UI_BIND`).
+- If `UI_USER`/`UI_PASS` are set, send HTTP basic auth on every request.
+- PostgreSQL: `docker compose exec -T postgres psql -U bucketeer -d bucketeer`
+  (or use `$DATABASE_URL` directly).
+
+Never write to the database or restart the stack; Bucketeer is the only writer.
+`docker compose down -v` **deletes all findings** — never use it here.
+
+## Vocabulary (answer carefully)
+
+- `exposure_state`: `unknown | public_dns | reachable | auth_required |
+  public_read | private | not_found`.
+- `public_read` = an anonymous read succeeded. That is a **fact**, not a claim
+  that all data is sensitive or safe.
+- `listable` = the provider's list API returned object names; only then is there
+  an index. `object_readable` = a known object was readable but listing is denied
+  (common for Azure "Blob" mode and CDN).
+- `reachable` (databases) = the port answered; **no authentication was
+  attempted**. A database is never reported readable.
+- Sources: `direct_ct`, `cname`, `manual`, `discovery`.
+
+## HTTP API (read)
+
+| Call | Answers |
+| --- | --- |
+| `GET /api/stats` | total assets, publicly readable, reachable, indexed items, breakdowns by provider/service/exposure |
+| `GET /api/assets?exposure=public_read&listable=true&limit=100` | open, enumerable buckets |
+| `GET /api/assets?provider=aws&service=s3&q=<term>&sort=exposure&limit=100&offset=0` | filtered/search asset list (envelope `{assets,total,limit,offset}`) |
+| `GET /api/entries?q=<term>&limit=100&offset=0` | search indexed object keys/paths (envelope `{entries,total,...}`) |
+| `GET /metrics` | Prometheus counters (ct_, dns_, probe_, bucket_candidates_, bucket_open_, db_, queue_) |
+| `GET /healthz`, `GET /readyz` | liveness / DB + monitor readiness |
+
+## HTTP API (actions — do only when the user asks)
+
+- `POST /api/scan` with JSON `{"target":"<host-or-URL>"}` — classify, follow
+  CNAME, probe and index a specific target. Azure container comes from the URL
+  path. Returns per-target `{provider,service,exposure,listable,entries,sample}`.
+- `POST /api/reprobe?exposure=reachable&limit=10000` — re-probe already-stored
+  assets with current checks and index any that are listable (async, 202).
+
+## Database (read-only) and the questions it answers
+
+Schema: `cloud_assets` (one row per provider/service/hostname), `asset_entries`
+(indexed keys), `observations` (sources, CNAME chain, IPs), `ct_checkpoints` and
+`ct_sths` (CT resume state; no contents).
+
+Open buckets with object counts:
+```sql
+SELECT a.provider, a.service, a.provider_hostname,
+       count(e.id) AS objects
+FROM cloud_assets a
+LEFT JOIN asset_entries e ON e.asset_id = a.id
+WHERE a.exposure_state = 'public_read' AND a.listable
+GROUP BY 1,2,3 ORDER BY objects DESC;
+```
+
+What is inside a bucket:
+```sql
+SELECT e.kind, e.name, e.size, e.updated
+FROM asset_entries e JOIN cloud_assets a ON a.id = e.asset_id
+WHERE a.provider_hostname = 'example.s3.amazonaws.com'
+ORDER BY e.name LIMIT 200;
+```
+
+Search indexed paths across everything:
+```sql
+SELECT a.provider, a.provider_hostname, e.name, e.size
+FROM asset_entries e JOIN cloud_assets a ON a.id = e.asset_id
+WHERE e.name ILIKE '%backup%' ORDER BY e.size DESC LIMIT 100;
+```
+
+Databases that answered (reachable != readable):
+```sql
+SELECT provider, service, provider_hostname, exposure_state, exposure_detail
+FROM cloud_assets
+WHERE service IN ('rds','postgres','mysql','redis','memorydb','elasticache','documentdb','neptune','cosmosdb')
+ORDER BY last_seen DESC LIMIT 100;
+```
+
+Object-readable but not listable (e.g. Azure "Blob" mode, CDN):
+```sql
+SELECT provider, provider_hostname, exposure_detail FROM cloud_assets
+WHERE object_readable AND NOT listable LIMIT 100;
+```
+
+## Answering rules
+
+1. Always filter on `listable` before claiming a bucket can be enumerated.
+2. Never equate `reachable` with readable; never claim data is "exposed" from a
+   hostname alone.
+3. If a counter is zero, say the discovery window found none yet — discovery is
+   continuous and stochastic, not a guarantee. Suggest `/api/reprobe` (re-check
+   stored assets) or `/api/scan` (test a specific target).
+4. Quote the stored `exposure_detail` as the factual reason.

internal/config/config.go

@@ -1,4 +1,4 @@
-// Package config loads all ctcloud configuration from environment variables.
+// Package config loads all bucketeer configuration from environment variables.
 // There are no configuration files at runtime.
 package config
 

internal/config/config_test.go

@@ -6,12 +6,12 @@ import (
 )
 
 func TestLoadDefaults(t *testing.T) {
-	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("DATABASE_URL", "postgres://localhost/bucketeer")
 	c, err := Load()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if c.DatabaseURL != "postgres://localhost/ctcloud" {
+	if c.DatabaseURL != "postgres://localhost/bucketeer" {
 		t.Fatalf("database url = %q", c.DatabaseURL)
 	}
 	if c.DNSWorkers != 64 || c.DNSMaxCNAMEdepth != 8 {
@@ -23,7 +23,7 @@ func TestLoadDefaults(t *testing.T) {
 }
 
 func TestLoadOverrides(t *testing.T) {
-	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("DATABASE_URL", "postgres://localhost/bucketeer")
 	t.Setenv("DNS_WORKERS", "4")
 	t.Setenv("ACTIVE_PROBING", "true")
 	t.Setenv("DB_FLUSH_INTERVAL", "250ms")
@@ -45,7 +45,7 @@ func TestLoadInvalid(t *testing.T) {
 	if _, err := Load(); err == nil {
 		t.Fatal("expected error without DATABASE_URL")
 	}
-	t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud")
+	t.Setenv("DATABASE_URL", "postgres://localhost/bucketeer")
 	t.Setenv("LOG_LEVEL", "verbose")
 	if _, err := Load(); err == nil {
 		t.Fatal("expected error for bad LOG_LEVEL")

internal/store/state.go

@@ -131,7 +131,7 @@ func (s *Store) RemoveSTH(ctx context.Context, logID monitor.LogID, sth *cttypes
 	return err
 }
 
-// StoreIssuer and LoadIssuer are no-ops. ctcloud's identifier sink bypasses
+// StoreIssuer and LoadIssuer are no-ops. bucketeer's identifier sink bypasses
 // certificate-chain retrieval entirely, so the issuer cache is never consulted;
 // keeping it empty also satisfies the "no certificate contents in the state
 // store" requirement. The Go runtime cache handles any incidental calls.
@@ -170,7 +170,7 @@ func (s *Store) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr e
 	return nil
 }
 
-// GetErrors returns the recent CT errors. ctcloud keeps only aggregate counters.
+// GetErrors returns the recent CT errors. bucketeer keeps only aggregate counters.
 func (s *Store) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
 	return "", nil
 }

internal/store/store.go

@@ -17,7 +17,7 @@ import (
 	"tea.karlcom.de/karl/bucketeer/migrations"
 )
 
-// Store wraps a pgx pool with the operations ctcloud needs.
+// Store wraps a pgx pool with the operations bucketeer needs.
 type Store struct {
 	pool    *pgxpool.Pool
 	metrics *metrics.Metrics