Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
146 lines
4.0 KiB
Go
146 lines
4.0 KiB
Go
package probe
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"io"
|
|
"net"
|
|
"testing"
|
|
"time"
|
|
|
|
"tea.karlcom.de/karl/bucketeer/internal/model"
|
|
)
|
|
|
|
func listenFake(t *testing.T, handle func(net.Conn)) (string, int) {
|
|
t.Helper()
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { ln.Close() })
|
|
go func() {
|
|
conn, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
handle(conn)
|
|
conn.Close()
|
|
}()
|
|
host, port := splitAddr(t, ln.Addr().String())
|
|
return host, port
|
|
}
|
|
|
|
func splitAddr(t *testing.T, addr string) (string, int) {
|
|
t.Helper()
|
|
host, portStr, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return host, mustAtoi(portStr)
|
|
}
|
|
|
|
func TestProbeRedisReadable(t *testing.T) {
|
|
host, port := listenFake(t, func(c net.Conn) {
|
|
buf := make([]byte, 6)
|
|
_, _ = io.ReadFull(c, buf)
|
|
_, _ = c.Write([]byte("+PONG\r\n"))
|
|
})
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
r, ok := p.probeRedis(context.Background(), host, port)
|
|
if !ok || r.Exposure != model.ExposurePublicRead || !r.ObjectReadable {
|
|
t.Fatalf("got %+v ok=%v, want public_read/object_readable", r, ok)
|
|
}
|
|
}
|
|
|
|
func TestProbeRedisAuthRequired(t *testing.T) {
|
|
host, port := listenFake(t, func(c net.Conn) {
|
|
buf := make([]byte, 6)
|
|
_, _ = io.ReadFull(c, buf)
|
|
_, _ = c.Write([]byte("-NOAUTH Authentication required.\r\n"))
|
|
})
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
r, _ := p.probeRedis(context.Background(), host, port)
|
|
if r.Exposure != model.ExposureAuthRequired {
|
|
t.Fatalf("got %+v, want auth_required", r)
|
|
}
|
|
}
|
|
|
|
func TestProbePostgresTrust(t *testing.T) {
|
|
host, port := listenFake(t, func(c net.Conn) {
|
|
ssl := make([]byte, 8)
|
|
if _, err := io.ReadFull(c, ssl); err != nil {
|
|
return
|
|
}
|
|
_, _ = c.Write([]byte{'N'})
|
|
var l [4]byte
|
|
if _, err := io.ReadFull(c, l[:]); err != nil {
|
|
return
|
|
}
|
|
n := int(binary.BigEndian.Uint32(l[:])) - 4
|
|
_, _ = io.ReadFull(c, make([]byte, n))
|
|
// AuthenticationOk: 'R', length 8, auth type 0
|
|
_, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 0})
|
|
})
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
r, _ := p.probePostgres(context.Background(), host, port)
|
|
if r.Exposure != model.ExposurePublicRead || !r.ObjectReadable {
|
|
t.Fatalf("got %+v, want public_read/object_readable", r)
|
|
}
|
|
}
|
|
|
|
func TestProbePostgresPassword(t *testing.T) {
|
|
host, port := listenFake(t, func(c net.Conn) {
|
|
ssl := make([]byte, 8)
|
|
if _, err := io.ReadFull(c, ssl); err != nil {
|
|
return
|
|
}
|
|
_, _ = c.Write([]byte{'N'})
|
|
var l [4]byte
|
|
if _, err := io.ReadFull(c, l[:]); err != nil {
|
|
return
|
|
}
|
|
n := int(binary.BigEndian.Uint32(l[:])) - 4
|
|
_, _ = io.ReadFull(c, make([]byte, n))
|
|
// AuthenticationMD5Password: 'R', length 8, auth type 5
|
|
_, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 5})
|
|
})
|
|
p := New(Config{Workers: 1, Timeout: time.Second})
|
|
r, _ := p.probePostgres(context.Background(), host, port)
|
|
if r.Exposure != model.ExposureAuthRequired {
|
|
t.Fatalf("got %+v, want auth_required", r)
|
|
}
|
|
}
|
|
|
|
func TestParseBSONFields(t *testing.T) {
|
|
var doc []byte
|
|
addDouble := func(name string, v uint64) {
|
|
doc = append(doc, 0x01)
|
|
doc = append(doc, appendCString(nil, name)...)
|
|
var tmp [8]byte
|
|
binary.LittleEndian.PutUint64(tmp[:], v)
|
|
doc = append(doc, tmp[:]...)
|
|
}
|
|
addString := func(name, val string) {
|
|
doc = append(doc, 0x02)
|
|
doc = append(doc, appendCString(nil, name)...)
|
|
var tmp [4]byte
|
|
binary.LittleEndian.PutUint32(tmp[:], uint32(len(val)+1))
|
|
doc = append(doc, tmp[:]...)
|
|
doc = append(doc, appendCString(nil, val)...)
|
|
}
|
|
addDouble("ok", 0x3FF0000000000000) // 1.0
|
|
addString("errmsg", "requires authentication")
|
|
body := make([]byte, 0, 5+len(doc)+5)
|
|
body = append(body, 0, 0, 0, 0, 0) // flags + section kind 0
|
|
full := make([]byte, 4, 4+len(doc)+1)
|
|
binary.LittleEndian.PutUint32(full, uint32(4+len(doc)+1))
|
|
full = append(full, doc...)
|
|
full = append(full, 0)
|
|
body = append(body, full...)
|
|
|
|
ok, errmsg := parseBSONFields(body)
|
|
if ok != 1 || errmsg != "requires authentication" {
|
|
t.Fatalf("got ok=%v errmsg=%q", ok, errmsg)
|
|
}
|
|
}
|