StasonJatham
62a7d2eb2e
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
87 lines
2.4 KiB
Go
87 lines
2.4 KiB
Go
package store
|
|
|
|
import (
|
|
"os"
|
|
"testing"
|
|
|
|
"software.sslmate.com/src/certspotter/cttypes"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
"tea.karlcom.de/karl/bucketeer/internal/pipeline"
|
|
)
|
|
|
|
func TestSTHKeyStableAndUnique(t *testing.T) {
|
|
a := &cttypes.SignedTreeHead{TreeSize: 10, Timestamp: 100}
|
|
if sthKey(a) != sthKey(a) {
|
|
t.Fatal("sthKey is not deterministic")
|
|
}
|
|
b := &cttypes.SignedTreeHead{TreeSize: 11, Timestamp: 100}
|
|
if sthKey(a) == sthKey(b) {
|
|
t.Fatal("sthKey collides for different tree sizes")
|
|
}
|
|
}
|
|
|
|
func TestNonNil(t *testing.T) {
|
|
if got := nonNil(nil); got == nil || len(got) != 0 {
|
|
t.Fatalf("nonNil(nil) = %#v", got)
|
|
}
|
|
in := []string{"a"}
|
|
if got := nonNil(in); len(got) != 1 || got[0] != "a" {
|
|
t.Fatalf("nonNil changed slice: %#v", got)
|
|
}
|
|
}
|
|
|
|
// TestIntegration exercises the real schema when DATABASE_URL is set.
|
|
func TestIntegration(t *testing.T) {
|
|
url := os.Getenv("DATABASE_URL")
|
|
if url == "" {
|
|
t.Skip("DATABASE_URL not set; skipping integration test")
|
|
}
|
|
ctx := t.Context()
|
|
s, err := Open(ctx, url, 2)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
if err := s.Migrate(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
f := pipeline.Finding{
|
|
Candidate: model.Candidate{
|
|
Hostname: "it-bucket.s3.eu-central-1.amazonaws.com",
|
|
Original: "it-bucket.s3.eu-central-1.amazonaws.com",
|
|
Provider: model.ProviderAWS,
|
|
Service: model.ServiceS3,
|
|
Source: model.SourceDirectCT,
|
|
},
|
|
IPv4: []string{"1.2.3.4"},
|
|
}
|
|
if err := s.WriteFindings(ctx, []pipeline.Finding{f}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.UpdateExposure(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, model.ExposurePublicRead, "test", true, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.ListAssets(ctx, AssetFilter{Provider: "aws", Limit: 10}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.CountAssets(ctx, AssetFilter{Provider: "aws"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.Stats(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := s.ReplaceEntries(ctx, model.ProviderAWS.String(), model.ServiceS3.String(), f.Hostname, []model.Entry{
|
|
{Kind: "object", Name: "it/index/a.txt", Size: 10},
|
|
{Kind: "prefix", Name: "it/index/"},
|
|
})
|
|
if err != nil || n != 2 {
|
|
t.Fatalf("ReplaceEntries = %d, %v", n, err)
|
|
}
|
|
entries, total, err := s.SearchEntries(ctx, "index/a", 10, 0)
|
|
if err != nil || total != 1 || len(entries) != 1 || entries[0].ProviderHost != f.Hostname {
|
|
t.Fatalf("SearchEntries = %+v total=%d err=%v", entries, total, err)
|
|
}
|
|
}
|