diff --git a/.dockerignore b/.dockerignore index d042cfe..4888dcd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,10 @@ .git .gitignore +.env pgdata *.test *.out .gocache +assets +docs +*.md diff --git a/.env.example b/.env.example index 822a1dd..a01cbcb 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,14 @@ # Copy to .env and edit. .env is gitignored. -POSTGRES_USER=ctcloud +POSTGRES_USER=bucketeer POSTGRES_PASSWORD=change-me-strong -POSTGRES_DB=ctcloud +POSTGRES_DB=bucketeer # Local compose (private bridge network, no TLS): -DATABASE_URL=postgres://ctcloud:change-me-strong@postgres:5432/ctcloud?sslmode=disable +DATABASE_URL=postgres://bucketeer:change-me-strong@postgres:5432/bucketeer?sslmode=disable # Production: use a TLS-enabled database instead, e.g. -# DATABASE_URL=postgres://user:pass@db.example.com:5432/ctcloud?sslmode=verify-full +# DATABASE_URL=postgres://user:pass@db.example.com:5432/bucketeer?sslmode=verify-full # DNS_SERVERS unset = ~20 curated public resolvers. Options: # DNS_SERVERS=system use the OS resolver diff --git a/.gitignore b/.gitignore index f647318..973cf0a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,14 @@ -/ctcloud +# Build output +/bucketeer +/dist/ *.test *.out -pgdata/ +coverage.* + +# Local configuration and data .env +pgdata/ + +# OS / editor +.DS_Store +*.swp diff --git a/Dockerfile b/Dockerfile index 5e13a4b..0d0e6b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,10 +13,10 @@ COPY . . # Static binary: no cgo, no shell, no writable filesystem required. RUN CGO_ENABLED=0 GOOS=linux go build \ -trimpath -ldflags="-s -w" \ - -o /out/ctcloud ./cmd/ctcloud + -o /out/bucketeer ./cmd/bucketeer FROM gcr.io/distroless/static-debian12:nonroot -COPY --from=build /out/ctcloud /ctcloud +COPY --from=build /out/bucketeer /bucketeer USER nonroot:nonroot EXPOSE 9090 -ENTRYPOINT ["/ctcloud"] +ENTRYPOINT ["/bucketeer"] diff --git a/README.md b/README.md index a0d5af0..1e403f8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,18 @@ -# ctcloud +

+ + + Bucketeer + +

-`ctcloud` continuously consumes Certificate Transparency (CT) logs, finds +# Bucketeer + +Bucketeer continuously consumes Certificate Transparency (CT) logs, finds cloud storage/database/data-service resources referenced by certificates, -enriches candidates through DNS/CNAME resolution, and persists only useful -findings to PostgreSQL. +enriches candidates through DNS/CNAME resolution, discovers and indexes open +buckets, and persists only useful findings to PostgreSQL. + +![Bucketeer UI](assets/bucketeer-landing.jpg) It is a **single stateless Go process plus PostgreSQL**. CertSpotter's real CT fetching, validation and parsing code is embedded and called **in-process**; @@ -11,7 +20,7 @@ there is no stdout handoff, no JSON handoff, no hooks, no subprocesses and no certificate files. > A detected provider endpoint **does not** mean the underlying data is publicly -> accessible. `ctcloud` reports facts (hostname existed in CT, resolved, answered +> accessible. `bucketeer` reports facts (hostname existed in CT, resolved, answered > a request) separately from conclusions (`exposure_state`). Most findings are > simply an endpoint that exists. @@ -38,7 +47,35 @@ CT logs ──► CertSpotter internals (fetch/verify/parse) PostgreSQL ``` -See [ARCHITECTURE.md](ARCHITECTURE.md) for per-stage rules. +See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for per-stage rules. + +## Repository layout + +```text +cmd/bucketeer/ application entry point +internal/ + classify/ zero-allocation provider/resource classifier + config/ environment configuration + discover/ bucket-name discovery from CT domains + dns/ resolver pool, TTL cache, CNAME following + index/ anonymous listing -> metadata index + metrics/ Prometheus text endpoint + model/ shared enums and structs + pipeline/ bounded queues, dedup, worker pools + probe/ reachability + readability checks + sample/ bounded random-sample queue + scan/ on-demand scan and backfill + store/ PostgreSQL schema, batch writer, CT state + webui/ embedded web UI +migrations/ embedded SQL schema +third_party/certspotter/ vendored upstream CT engine (minimal patch) +docs/ ARCHITECTURE.md, DESIGN.md, GOAL.md, SKILL.md +assets/ brand assets +``` + +More: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) (data flow), +[docs/DESIGN.md](docs/DESIGN.md) (UI design system), +[docs/SKILL.md](docs/SKILL.md) (how an agent queries and audits a running instance). ## Why CertSpotter is embedded (and patched) @@ -48,7 +85,7 @@ tree maintenance, entry parsing and certificate reconstruction. The only change is a two-hunk patch in `third_party/certspotter` that adds an `IdentifierSink` callback. After a certificate is parsed, the identifiers are -handed to ctcloud with a direct Go function call and the watchlist/chain/ +handed to bucketeer with a direct Go function call and the watchlist/chain/ notification machinery is skipped: ```go @@ -147,7 +184,7 @@ cp .env.example .env # set a strong POSTGRES_PASSWORD and matching DATABASE docker compose up --build ``` -That starts `postgres` and the stateless `ctcloud` container. The application +That starts `postgres` and the stateless `bucketeer` container. The application container runs as non-root, has a read-only root filesystem, drops all Linux capabilities, sets CPU/memory/pids limits and mounts no application volume. The only persistent state is PostgreSQL (CT checkpoints + cloud assets + index). @@ -169,7 +206,7 @@ basic auth on the UI and its API. ## Web UI A small, dependency-free UI (vanilla JS, no build step, embedded in the binary) -styled to the tokens and layout in `DESIGN.md` (neutral canvas, single 768px +styled to the tokens and layout in [docs/DESIGN.md](docs/DESIGN.md) (neutral canvas, single 768px results column, flat rows, pill controls, no shadows). * search box with 250ms debounce, stale-request cancellation and `?q=` URL state @@ -201,8 +238,8 @@ stay crisp on HiDPI displays while shrinking the ~1.2 MB source PNGs by ~96%. Regenerate after replacing the source PNGs at the repo root: ```sh -magick bucketeer-light.png -resize 256x -strip /tmp/l.png -magick bucketeer-dark.png -resize 256x -strip /tmp/d.png +magick assets/bucketeer-light.png -resize 256x -strip /tmp/l.png +magick assets/bucketeer-dark.png -resize 256x -strip /tmp/d.png cwebp -quiet -lossless -z 9 -alpha_q 100 /tmp/l.png -o internal/webui/assets/logo-light.webp cwebp -quiet -lossless -z 9 -alpha_q 100 /tmp/d.png -o internal/webui/assets/logo-dark.webp ``` @@ -249,7 +286,7 @@ attempted`). Database endpoints are never claimed to be readable: a raw * **Database TLS** — the local compose stack uses `sslmode=disable` because the compose bridge network is private. For production, point `DATABASE_URL` at a TLS-enabled database, e.g. - `postgres://user:pass@db.example.com:5432/ctcloud?sslmode=verify-full`. + `postgres://user:pass@db.example.com:5432/bucketeer?sslmode=verify-full`. pgx handles TLS natively; no extra flags are needed. * **Metrics** — bound to loopback. There is no auth, no debug endpoints and no high-cardinality labels (never hostnames). @@ -367,7 +404,7 @@ go vet ./... Integration tests for PostgreSQL run only when `DATABASE_URL` is set: ```sh -DATABASE_URL=postgres://ctcloud:ctcloud@localhost:5432/ctcloud go test ./internal/store/ +DATABASE_URL=postgres://bucketeer:bucketeer@localhost:5432/bucketeer go test ./internal/store/ ``` ## Limitations diff --git a/bucketeer-dark.png b/assets/bucketeer-dark.png similarity index 100% rename from bucketeer-dark.png rename to assets/bucketeer-dark.png diff --git a/bucketeer-landing.jpg b/assets/bucketeer-landing.jpg similarity index 100% rename from bucketeer-landing.jpg rename to assets/bucketeer-landing.jpg diff --git a/bucketeer-light.png b/assets/bucketeer-light.png similarity index 100% rename from bucketeer-light.png rename to assets/bucketeer-light.png diff --git a/cmd/ctcloud/main.go b/cmd/bucketeer/main.go similarity index 99% rename from cmd/ctcloud/main.go rename to cmd/bucketeer/main.go index d6afc55..44d962a 100644 --- a/cmd/ctcloud/main.go +++ b/cmd/bucketeer/main.go @@ -1,4 +1,4 @@ -// Command ctcloud continuously consumes Certificate Transparency logs through +// Command bucketeer continuously consumes Certificate Transparency logs through // embedded CertSpotter internals, classifies cloud/database/storage resources // in-process and persists only useful, enriched findings to PostgreSQL. package main @@ -242,7 +242,7 @@ func run() error { httpSrv := startHTTP(cfg.MetricsAddr, m, st, ui, &monitorReady) go logStats(ctx, m, p, st, resolver, prober, cfg.StatsInterval) - slog.Info("ctcloud started") + slog.Info("bucketeer started") err = <-monitorDone if err != nil && !errors.Is(err, context.Canceled) { slog.Error("ct monitor stopped", "error", err) diff --git a/docker-compose.yml b/docker-compose.yml index 9c686f8..76a4ba6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ # Secrets come from .env (gitignored). Copy .env.example to .env and set a # strong POSTGRES_PASSWORD plus a matching DATABASE_URL before `docker compose # up`. For a production database, point DATABASE_URL at a TLS-enabled endpoint, -# e.g. postgres://user:pass@db.example.com:5432/ctcloud?sslmode=verify-full +# e.g. postgres://user:pass@db.example.com:5432/bucketeer?sslmode=verify-full # # The compose bridge network is private; sslmode=disable is only acceptable here # because the database is not reachable from outside the stack. @@ -11,13 +11,13 @@ services: postgres: image: postgres:17-alpine environment: - POSTGRES_USER: ${POSTGRES_USER:-ctcloud} + POSTGRES_USER: ${POSTGRES_USER:-bucketeer} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} - POSTGRES_DB: ${POSTGRES_DB:-ctcloud} + POSTGRES_DB: ${POSTGRES_DB:-bucketeer} volumes: - pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-ctcloud} -d ${POSTGRES_DB:-ctcloud}"] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-bucketeer} -d ${POSTGRES_DB:-bucketeer}"] interval: 5s timeout: 5s retries: 12 @@ -29,13 +29,13 @@ services: options: { max-size: "10m", max-file: "3" } restart: unless-stopped - ctcloud: + bucketeer: build: . depends_on: postgres: condition: service_healthy - # All optional tuning (DNS_SERVERS, PROBE_*, UI_USER, ...) comes from .env. - # Leave DNS_SERVERS unset to use the curated + community resolver list. + # All optional tuning (DNS_SERVERS, PROBE_*, BUCKET_SCAN_*, UI_USER, ...) + # comes from .env. Leave DNS_SERVERS unset to use the curated resolvers. env_file: - .env environment: diff --git a/ARCHITECTURE.md b/docs/ARCHITECTURE.md similarity index 97% rename from ARCHITECTURE.md rename to docs/ARCHITECTURE.md index ea62dca..e6d47b2 100644 --- a/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -66,7 +66,7 @@ Certificate Transparency Logs ## Failure handling * DNS and CT network errors are retried by CertSpotter with bounded exponential - backoff; ctcloud counts them (`ct_errors_total`) without logging per event. + backoff; bucketeer counts them (`ct_errors_total`) without logging per event. * PostgreSQL writes retry a few times with backoff; persistent failure drops items from the bounded DB queue rather than growing memory. * The process uses `context` throughout and on `SIGTERM` stops intake, drains diff --git a/DESIGN.md b/docs/DESIGN.md similarity index 100% rename from DESIGN.md rename to docs/DESIGN.md diff --git a/GOAL.md b/docs/GOAL.md similarity index 99% rename from GOAL.md rename to docs/GOAL.md index d6731ea..10a2e67 100644 --- a/GOAL.md +++ b/docs/GOAL.md @@ -1157,7 +1157,7 @@ docker-compose.yml Docker Compose should contain: ```text -ctcloud +bucketeer postgres ``` @@ -1172,7 +1172,7 @@ Keep the project simple. Suggested layout: ```text -cmd/ctcloud/ +cmd/bucketeer/ main.go internal/ diff --git a/docs/SKILL.md b/docs/SKILL.md new file mode 100644 index 0000000..94a4f4a --- /dev/null +++ b/docs/SKILL.md @@ -0,0 +1,110 @@ +--- +name: bucketeer-ops +description: Query and audit a running Bucketeer instance (CT cloud-resource monitor) over its HTTP API and PostgreSQL, and answer user questions about discovered assets, open buckets, indexed objects and databases. +--- + +# Operating and auditing Bucketeer + +Bucketeer is a single Go process + PostgreSQL that consumes CT logs, classifies +cloud resources, discovers open buckets, and indexes their contents. This skill is +for an agent that needs to **answer questions from the data** without reading the +source. + +## Access + +Assume a local deployment (`docker compose`): + +- API base: `http://127.0.0.1:9090` (override with `METRICS_ADDR` / `UI_BIND`). +- If `UI_USER`/`UI_PASS` are set, send HTTP basic auth on every request. +- PostgreSQL: `docker compose exec -T postgres psql -U bucketeer -d bucketeer` + (or use `$DATABASE_URL` directly). + +Never write to the database or restart the stack; Bucketeer is the only writer. +`docker compose down -v` **deletes all findings** — never use it here. + +## Vocabulary (answer carefully) + +- `exposure_state`: `unknown | public_dns | reachable | auth_required | + public_read | private | not_found`. +- `public_read` = an anonymous read succeeded. That is a **fact**, not a claim + that all data is sensitive or safe. +- `listable` = the provider's list API returned object names; only then is there + an index. `object_readable` = a known object was readable but listing is denied + (common for Azure "Blob" mode and CDN). +- `reachable` (databases) = the port answered; **no authentication was + attempted**. A database is never reported readable. +- Sources: `direct_ct`, `cname`, `manual`, `discovery`. + +## HTTP API (read) + +| Call | Answers | +| --- | --- | +| `GET /api/stats` | total assets, publicly readable, reachable, indexed items, breakdowns by provider/service/exposure | +| `GET /api/assets?exposure=public_read&listable=true&limit=100` | open, enumerable buckets | +| `GET /api/assets?provider=aws&service=s3&q=&sort=exposure&limit=100&offset=0` | filtered/search asset list (envelope `{assets,total,limit,offset}`) | +| `GET /api/entries?q=&limit=100&offset=0` | search indexed object keys/paths (envelope `{entries,total,...}`) | +| `GET /metrics` | Prometheus counters (ct_, dns_, probe_, bucket_candidates_, bucket_open_, db_, queue_) | +| `GET /healthz`, `GET /readyz` | liveness / DB + monitor readiness | + +## HTTP API (actions — do only when the user asks) + +- `POST /api/scan` with JSON `{"target":""}` — classify, follow + CNAME, probe and index a specific target. Azure container comes from the URL + path. Returns per-target `{provider,service,exposure,listable,entries,sample}`. +- `POST /api/reprobe?exposure=reachable&limit=10000` — re-probe already-stored + assets with current checks and index any that are listable (async, 202). + +## Database (read-only) and the questions it answers + +Schema: `cloud_assets` (one row per provider/service/hostname), `asset_entries` +(indexed keys), `observations` (sources, CNAME chain, IPs), `ct_checkpoints` and +`ct_sths` (CT resume state; no contents). + +Open buckets with object counts: +```sql +SELECT a.provider, a.service, a.provider_hostname, + count(e.id) AS objects +FROM cloud_assets a +LEFT JOIN asset_entries e ON e.asset_id = a.id +WHERE a.exposure_state = 'public_read' AND a.listable +GROUP BY 1,2,3 ORDER BY objects DESC; +``` + +What is inside a bucket: +```sql +SELECT e.kind, e.name, e.size, e.updated +FROM asset_entries e JOIN cloud_assets a ON a.id = e.asset_id +WHERE a.provider_hostname = 'example.s3.amazonaws.com' +ORDER BY e.name LIMIT 200; +``` + +Search indexed paths across everything: +```sql +SELECT a.provider, a.provider_hostname, e.name, e.size +FROM asset_entries e JOIN cloud_assets a ON a.id = e.asset_id +WHERE e.name ILIKE '%backup%' ORDER BY e.size DESC LIMIT 100; +``` + +Databases that answered (reachable != readable): +```sql +SELECT provider, service, provider_hostname, exposure_state, exposure_detail +FROM cloud_assets +WHERE service IN ('rds','postgres','mysql','redis','memorydb','elasticache','documentdb','neptune','cosmosdb') +ORDER BY last_seen DESC LIMIT 100; +``` + +Object-readable but not listable (e.g. Azure "Blob" mode, CDN): +```sql +SELECT provider, provider_hostname, exposure_detail FROM cloud_assets +WHERE object_readable AND NOT listable LIMIT 100; +``` + +## Answering rules + +1. Always filter on `listable` before claiming a bucket can be enumerated. +2. Never equate `reachable` with readable; never claim data is "exposed" from a + hostname alone. +3. If a counter is zero, say the discovery window found none yet — discovery is + continuous and stochastic, not a guarantee. Suggest `/api/reprobe` (re-check + stored assets) or `/api/scan` (test a specific target). +4. Quote the stored `exposure_detail` as the factual reason. diff --git a/internal/config/config.go b/internal/config/config.go index b06fe64..fc04c78 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,4 +1,4 @@ -// Package config loads all ctcloud configuration from environment variables. +// Package config loads all bucketeer configuration from environment variables. // There are no configuration files at runtime. package config diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6a78a41..0bec2c8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,12 +6,12 @@ import ( ) func TestLoadDefaults(t *testing.T) { - t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud") + t.Setenv("DATABASE_URL", "postgres://localhost/bucketeer") c, err := Load() if err != nil { t.Fatal(err) } - if c.DatabaseURL != "postgres://localhost/ctcloud" { + if c.DatabaseURL != "postgres://localhost/bucketeer" { t.Fatalf("database url = %q", c.DatabaseURL) } if c.DNSWorkers != 64 || c.DNSMaxCNAMEdepth != 8 { @@ -23,7 +23,7 @@ func TestLoadDefaults(t *testing.T) { } func TestLoadOverrides(t *testing.T) { - t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud") + t.Setenv("DATABASE_URL", "postgres://localhost/bucketeer") t.Setenv("DNS_WORKERS", "4") t.Setenv("ACTIVE_PROBING", "true") t.Setenv("DB_FLUSH_INTERVAL", "250ms") @@ -45,7 +45,7 @@ func TestLoadInvalid(t *testing.T) { if _, err := Load(); err == nil { t.Fatal("expected error without DATABASE_URL") } - t.Setenv("DATABASE_URL", "postgres://localhost/ctcloud") + t.Setenv("DATABASE_URL", "postgres://localhost/bucketeer") t.Setenv("LOG_LEVEL", "verbose") if _, err := Load(); err == nil { t.Fatal("expected error for bad LOG_LEVEL") diff --git a/internal/store/state.go b/internal/store/state.go index e7977d8..423ddaa 100644 --- a/internal/store/state.go +++ b/internal/store/state.go @@ -131,7 +131,7 @@ func (s *Store) RemoveSTH(ctx context.Context, logID monitor.LogID, sth *cttypes return err } -// StoreIssuer and LoadIssuer are no-ops. ctcloud's identifier sink bypasses +// StoreIssuer and LoadIssuer are no-ops. bucketeer's identifier sink bypasses // certificate-chain retrieval entirely, so the issuer cache is never consulted; // keeping it empty also satisfies the "no certificate contents in the state // store" requirement. The Go runtime cache handles any incidental calls. @@ -170,7 +170,7 @@ func (s *Store) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr e return nil } -// GetErrors returns the recent CT errors. ctcloud keeps only aggregate counters. +// GetErrors returns the recent CT errors. bucketeer keeps only aggregate counters. func (s *Store) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) { return "", nil } diff --git a/internal/store/store.go b/internal/store/store.go index 0b06aa3..ea7debc 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,7 +17,7 @@ import ( "tea.karlcom.de/karl/bucketeer/migrations" ) -// Store wraps a pgx pool with the operations ctcloud needs. +// Store wraps a pgx pool with the operations bucketeer needs. type Store struct { pool *pgxpool.Pool metrics *metrics.Metrics