Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs
Logging: - structured slog with LOG_FORMAT=json|text and a stable key vocabulary - periodic stats record with queue depths, db pool, goroutines, memory, dns/probe - no per-certificate logging (hot path is counters only) UI (Bucketeer branding, DESIGN.md): - light+dark themes (OS default, persisted header toggle, no first-paint flash) - official mascot: lossless WebP light/dark logos + favicon, embedded and served with cache headers (1.2MB source PNGs -> ~27/22KB webp) - Assets/Contents views, pagination (offset) and native virtual scrolling (content-visibility) so large result sets do not blow up the DOM - search with debounce/abort + URL state, exposure chips, sort, summary counts Readability + indexing: - internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch (metadata only: keys/sizes/dates; never object contents) - asset_entries table + store write/search; readable resources are indexed and searchable in the UI - deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING, PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune HTTP reads - Azure account-level listing fixed: 403 is reported reachable, not auth_required, since anonymous access is per-container DNS: - curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at once broke the container resolver) - random healthy selection + failover + per-server cooldown Other: prober Close fixes a shutdown hang; config/README updated.
This commit is contained in:
StasonJatham committed
1 parent
8fce55f8c9
commit
a366905a43
28 files changed
+9744
-241
No files matched your search
+61
-2
@@ -23,6 +23,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/index"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/metrics"
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
@@ -34,14 +35,21 @@ type Config struct {
|
||||
Workers int
|
||||
Timeout time.Duration
|
||||
Queue int
|
||||
Deep bool // unauthenticated protocol reads for database services
|
||||
Metrics *metrics.Metrics
|
||||
Update UpdateFunc
|
||||
|
||||
// Indexer, when set, lists metadata for resources confirmed public_read and
|
||||
// hands the entries to OnIndex. Contents are never downloaded.
|
||||
Indexer *index.Indexer
|
||||
OnIndex func(ctx context.Context, c model.Candidate, entries []model.Entry)
|
||||
}
|
||||
|
||||
type Prober struct {
|
||||
cfg Config
|
||||
sample *sampleQueue
|
||||
client *http.Client
|
||||
stop chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
@@ -75,6 +83,12 @@ func (q *sampleQueue) push(c model.Candidate) {
|
||||
}
|
||||
}
|
||||
|
||||
func (q *sampleQueue) len() int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return len(q.items)
|
||||
}
|
||||
|
||||
func (q *sampleQueue) pop() (model.Candidate, bool) {
|
||||
q.mu.Lock()
|
||||
n := len(q.items)
|
||||
@@ -103,6 +117,7 @@ func New(cfg Config) *Prober {
|
||||
return &Prober{
|
||||
cfg: cfg,
|
||||
sample: newSampleQueue(cfg.Queue),
|
||||
stop: make(chan struct{}),
|
||||
client: &http.Client{
|
||||
Timeout: cfg.Timeout,
|
||||
// Observe the first response; do not follow redirects so region
|
||||
@@ -141,14 +156,23 @@ func (p *Prober) Submit(c model.Candidate) {
|
||||
p.sample.push(c)
|
||||
}
|
||||
|
||||
// Close stops the workers after any in-flight probe. It must be called before
|
||||
// Wait, otherwise Wait blocks until ctx is cancelled.
|
||||
func (p *Prober) Close() { close(p.stop) }
|
||||
|
||||
func (p *Prober) Wait() { p.wg.Wait() }
|
||||
|
||||
// QueueLen reports the current probe sample depth.
|
||||
func (p *Prober) QueueLen() int { return p.sample.len() }
|
||||
|
||||
func (p *Prober) worker(ctx context.Context) {
|
||||
defer p.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-p.stop:
|
||||
return
|
||||
case <-p.sample.notify:
|
||||
}
|
||||
for {
|
||||
@@ -163,12 +187,27 @@ func (p *Prober) worker(ctx context.Context) {
|
||||
if p.cfg.Update != nil {
|
||||
p.cfg.Update(ctx, c, state, detail)
|
||||
}
|
||||
if state == model.ExposurePublicRead && p.cfg.Indexer != nil && p.cfg.OnIndex != nil {
|
||||
if entries, err := p.cfg.Indexer.List(ctx, c); err == nil && len(entries) > 0 {
|
||||
p.cfg.OnIndex(ctx, c, entries)
|
||||
if p.cfg.Metrics != nil {
|
||||
p.cfg.Metrics.EntriesIndexed.Add(uint64(len(entries)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Probe runs the vendor-appropriate check for a single candidate.
|
||||
func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
// Deep, read-only protocol checks take precedence for database services and
|
||||
// can distinguish "reachable" from "actually readable without credentials".
|
||||
if p.cfg.Deep {
|
||||
if state, detail, ok := p.deepDB(ctx, c); ok {
|
||||
return state, detail
|
||||
}
|
||||
}
|
||||
switch c.Provider {
|
||||
case model.ProviderAWS:
|
||||
switch c.Service {
|
||||
@@ -188,8 +227,7 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
|
||||
case model.ProviderAzure:
|
||||
switch c.Service {
|
||||
case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
|
||||
// Anonymous container/file/share/queue/table listing.
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/?comp=list")
|
||||
return p.checkAzureStorage(ctx, c)
|
||||
case model.ServiceWebApp, model.ServiceSearch:
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
case model.ServiceCosmosDB:
|
||||
@@ -230,6 +268,27 @@ func (p *Prober) Probe(ctx context.Context, c model.Candidate) (model.ExposureSt
|
||||
return p.checkHTTP(ctx, "https://"+c.Hostname+"/")
|
||||
}
|
||||
|
||||
// checkAzureStorage probes the account-level container list. Anonymous read is
|
||||
// configured per container in Azure ("Public access level"), never at the
|
||||
// account level, so a 401/403 here means only that account listing needs auth —
|
||||
// it does NOT mean the containers are unreadable. We therefore report reachable
|
||||
// rather than auth_required, and only index if account listing is unexpectedly
|
||||
// public (which would let us discover container names).
|
||||
func (p *Prober) checkAzureStorage(ctx context.Context, c model.Candidate) (model.ExposureState, string) {
|
||||
status, _, err := p.do(ctx, "https://"+c.Hostname+"/?comp=list")
|
||||
if err != nil {
|
||||
return classifyError(err)
|
||||
}
|
||||
switch {
|
||||
case status >= 200 && status < 300:
|
||||
return model.ExposurePublicRead, "azure account-level anonymous listing succeeded"
|
||||
case status == http.StatusUnauthorized || status == http.StatusForbidden:
|
||||
return model.ExposureReachable, "azure account listing requires auth; per-container readability needs a container name"
|
||||
default:
|
||||
return classifyStatus(status, "azure")
|
||||
}
|
||||
}
|
||||
|
||||
// checkS3 queries the S3 ListObjectsV2 anonymous endpoint. A 200 means the
|
||||
// bucket allows anonymous listing (public read); 403 means it exists but is not
|
||||
// public; 404 means no such bucket. 301/307 carries the real region.
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
package probe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
// Deep, unauthenticated, read-only protocol probes. They determine whether a
|
||||
// database endpoint actually answers a read without credentials. No password,
|
||||
// token or credential of any kind is ever sent, and no state is mutated.
|
||||
|
||||
func (p *Prober) deepDB(ctx context.Context, c model.Candidate) (model.ExposureState, string, bool) {
|
||||
switch c.Service {
|
||||
case model.ServiceRedis, model.ServiceElastiCache, model.ServiceMemoryDB:
|
||||
return p.probeRedis(ctx, c.Hostname, dbPort(c.Service))
|
||||
case model.ServiceRDS, model.ServicePostgres, model.ServiceRedshift:
|
||||
return p.probePostgres(ctx, c.Hostname, 5432)
|
||||
case model.ServiceDocumentDB, model.ServiceCosmosDB:
|
||||
return p.probeMongo(ctx, c.Hostname, 27017)
|
||||
case model.ServiceOpenSearch:
|
||||
return p.probeHTTPReadable(ctx, "https://"+c.Hostname+"/_cat/indices?format=json&h=index")
|
||||
case model.ServiceNeptune:
|
||||
return p.probeHTTPReadable(ctx, "https://"+c.Hostname+"/status")
|
||||
}
|
||||
return model.ExposureUnknown, "", false
|
||||
}
|
||||
|
||||
func dbPort(s model.Service) int {
|
||||
switch s {
|
||||
case model.ServiceMySQL:
|
||||
return 3306
|
||||
case model.ServicePostgres, model.ServiceRDS:
|
||||
return 5432
|
||||
case model.ServiceRedis, model.ServiceElastiCache, model.ServiceMemoryDB:
|
||||
return 6379
|
||||
case model.ServiceDocumentDB, model.ServiceCosmosDB:
|
||||
return 27017
|
||||
case model.ServiceRedshift:
|
||||
return 5439
|
||||
case model.ServiceNeptune:
|
||||
return 8182
|
||||
case model.ServiceDatabase:
|
||||
return 1433
|
||||
default:
|
||||
return 5432
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Prober) dial(ctx context.Context, host string, port int) (net.Conn, error) {
|
||||
d := net.Dialer{Timeout: p.cfg.Timeout}
|
||||
return d.DialContext(ctx, "tcp", net.JoinHostPort(host, fmt.Sprint(port)))
|
||||
}
|
||||
|
||||
func (p *Prober) dialTLS(ctx context.Context, host string, port int) (net.Conn, error) {
|
||||
conn, err := p.dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tconn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true, ServerName: host}) // observation only
|
||||
_ = tconn.SetDeadline(time.Now().Add(p.cfg.Timeout))
|
||||
if err := tconn.HandshakeContext(ctx); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
return tconn, nil
|
||||
}
|
||||
|
||||
// probeRedis sends PING. Redis answers +PONG when no authentication is required.
|
||||
func (p *Prober) probeRedis(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
|
||||
var lastErr error
|
||||
for _, tlsOn := range []bool{false, true} {
|
||||
var (
|
||||
conn net.Conn
|
||||
err error
|
||||
)
|
||||
if tlsOn {
|
||||
conn, err = p.dialTLS(ctx, host, port)
|
||||
} else {
|
||||
conn, err = p.dial(ctx, host, port)
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
_ = conn.SetDeadline(time.Now().Add(p.cfg.Timeout))
|
||||
_, _ = conn.Write([]byte("PING\r\n"))
|
||||
buf := make([]byte, 256)
|
||||
n, _ := conn.Read(buf)
|
||||
conn.Close()
|
||||
resp := string(buf[:n])
|
||||
switch {
|
||||
case strings.HasPrefix(resp, "+PONG"):
|
||||
return model.ExposurePublicRead, "redis PING answered without auth", true
|
||||
case strings.Contains(resp, "NOAUTH"), strings.HasPrefix(resp, "-ERR"), strings.Contains(resp, "DENIED"), strings.Contains(resp, "protected mode"):
|
||||
return model.ExposureAuthRequired, "redis requires authentication", true
|
||||
}
|
||||
// no/invalid response: try the other transport
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("redis: no response")
|
||||
}
|
||||
st, detail := classifyError(lastErr)
|
||||
return st, detail, true
|
||||
}
|
||||
|
||||
// probePostgres negotiates TLS if offered, then sends a StartupMessage with no
|
||||
// password. The server's AuthenticationRequest tells us whether it trusts the
|
||||
// connection (AuthenticationOk) or requires a password.
|
||||
func (p *Prober) probePostgres(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
|
||||
conn, err := p.dial(ctx, host, port)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(p.cfg.Timeout))
|
||||
|
||||
// SSLRequest (length 8, code 80877103).
|
||||
req := []byte{0, 0, 0, 8, 4, 210, 22, 47}
|
||||
if _, err := conn.Write(req); err == nil {
|
||||
one := make([]byte, 1)
|
||||
if _, err := io.ReadFull(conn, one); err == nil {
|
||||
switch one[0] {
|
||||
case 'S': // proceed with TLS
|
||||
tconn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true, ServerName: host})
|
||||
if err := tconn.HandshakeContext(ctx); err == nil {
|
||||
conn = tconn
|
||||
}
|
||||
case 'N': // plaintext only
|
||||
case 'E': // server refused; fall through and try startup anyway
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StartupMessage: user=postgres database=postgres
|
||||
payload := []byte{0, 3, 0, 0}
|
||||
payload = append(payload, "user\x00postgres\x00database\x00postgres\x00\x00"...)
|
||||
msg := make([]byte, 4+len(payload))
|
||||
binary.BigEndian.PutUint32(msg, uint32(4+len(payload)))
|
||||
copy(msg[4:], payload)
|
||||
if _, err := conn.Write(msg); err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
|
||||
head := make([]byte, 5)
|
||||
if _, err := io.ReadFull(conn, head); err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
kind := head[0]
|
||||
length := int(binary.BigEndian.Uint32(head[1:5]))
|
||||
if length > 4 {
|
||||
body := make([]byte, length-4)
|
||||
_, _ = io.ReadFull(conn, body)
|
||||
if kind == 'R' && len(body) >= 4 {
|
||||
authType := binary.BigEndian.Uint32(body[:4])
|
||||
if authType == 0 {
|
||||
return model.ExposurePublicRead, "postgres AuthenticationOk (trust, no password required)", true
|
||||
}
|
||||
return model.ExposureAuthRequired, fmt.Sprintf("postgres requires password (auth type %d)", authType), true
|
||||
}
|
||||
if kind == 'E' {
|
||||
return model.ExposureAuthRequired, "postgres rejected startup (pg_hba/host not permitted)", true
|
||||
}
|
||||
}
|
||||
return model.ExposureReachable, "postgres port open; auth state unclear", true
|
||||
}
|
||||
|
||||
// probeMongo performs the OP_MSG handshake then a read-only listDatabases. A
|
||||
// successful, non-empty reply means the deployment has no authentication.
|
||||
func (p *Prober) probeMongo(ctx context.Context, host string, port int) (model.ExposureState, string, bool) {
|
||||
var (
|
||||
conn net.Conn
|
||||
err error
|
||||
)
|
||||
// Managed MongoDB (DocumentDB, Cosmos) requires TLS; try it, then plain.
|
||||
conn, err = p.dialTLS(ctx, host, port)
|
||||
if err != nil {
|
||||
conn, err = p.dial(ctx, host, port)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(p.cfg.Timeout))
|
||||
|
||||
doc := bsonDoc{"listDatabases": int32(1), "$db": "admin"}
|
||||
if err := writeOPMsg(conn, doc); err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
raw, err := readOPMsg(conn)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
ok, errmsg := parseBSONFields(raw)
|
||||
switch {
|
||||
case ok == 1 && errmsg == "":
|
||||
return model.ExposurePublicRead, "mongodb listDatabases succeeded without auth", true
|
||||
case strings.Contains(strings.ToLower(errmsg), "auth") || strings.Contains(strings.ToLower(errmsg), "unauthorized"):
|
||||
return model.ExposureAuthRequired, "mongodb requires authentication", true
|
||||
default:
|
||||
return model.ExposureReachable, "mongodb replied but auth state unclear", true
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Prober) probeHTTPReadable(ctx context.Context, endpoint string) (model.ExposureState, string, bool) {
|
||||
status, _, err := p.do(ctx, endpoint)
|
||||
if err != nil {
|
||||
st, d := classifyError(err)
|
||||
return st, d, true
|
||||
}
|
||||
st, d := classifyStatus(status, "")
|
||||
return st, d, true
|
||||
}
|
||||
|
||||
// --- minimal BSON/OP_MSG helpers (read-only) ---
|
||||
|
||||
type bsonDoc map[string]any
|
||||
|
||||
func appendCString(b []byte, s string) []byte {
|
||||
b = append(b, s...)
|
||||
return append(b, 0)
|
||||
}
|
||||
|
||||
func encodeBSON(doc bsonDoc) []byte {
|
||||
var body []byte
|
||||
for k, v := range doc {
|
||||
switch val := v.(type) {
|
||||
case int32:
|
||||
body = append(body, 0x10)
|
||||
body = appendCString(body, k)
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(val))
|
||||
body = append(body, tmp[:]...)
|
||||
case string:
|
||||
body = append(body, 0x02)
|
||||
body = appendCString(body, k)
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(len(val)+1))
|
||||
body = append(body, tmp[:]...)
|
||||
body = appendCString(body, val)
|
||||
}
|
||||
}
|
||||
out := make([]byte, 4, 4+len(body)+1)
|
||||
binary.LittleEndian.PutUint32(out, uint32(4+len(body)+1))
|
||||
out = append(out, body...)
|
||||
out = append(out, 0)
|
||||
return out
|
||||
}
|
||||
|
||||
func writeOPMsg(w io.Writer, doc bsonDoc) error {
|
||||
bson := encodeBSON(doc)
|
||||
msg := make([]byte, 0, 16+4+1+len(bson))
|
||||
var hdr [16]byte
|
||||
binary.LittleEndian.PutUint32(hdr[0:], uint32(16+4+1+len(bson))) // messageLength
|
||||
binary.LittleEndian.PutUint32(hdr[4:], 1) // requestID
|
||||
binary.LittleEndian.PutUint32(hdr[8:], 0) // responseTo
|
||||
binary.LittleEndian.PutUint32(hdr[12:], 2013) // opCode OP_MSG
|
||||
msg = append(msg, hdr[:]...)
|
||||
msg = append(msg, 0, 0, 0, 0) // flagBits
|
||||
msg = append(msg, 0) // section kind 0
|
||||
msg = append(msg, bson...)
|
||||
_, err := w.Write(msg)
|
||||
return err
|
||||
}
|
||||
|
||||
func readOPMsg(r io.Reader) ([]byte, error) {
|
||||
hdr := make([]byte, 16)
|
||||
if _, err := io.ReadFull(r, hdr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
length := int(binary.LittleEndian.Uint32(hdr[0:]))
|
||||
if length < 21 || length > 8<<20 {
|
||||
return nil, fmt.Errorf("mongo: bad message length %d", length)
|
||||
}
|
||||
body := make([]byte, length-16)
|
||||
if _, err := io.ReadFull(r, body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// parseBSONFields extracts the top-level "ok" (double/int) and "errmsg" fields.
|
||||
func parseBSONFields(b []byte) (float64, string) {
|
||||
if len(b) < 5 {
|
||||
return 0, ""
|
||||
}
|
||||
// body: flagBits(4) sectionKind(1) [doc] or [kind 1 + size + id + doc]
|
||||
p := 4
|
||||
if p < len(b) && b[p] == 1 && p+5 <= len(b) {
|
||||
p++ // section kind 1
|
||||
sz := int(binary.LittleEndian.Uint32(b[p:]))
|
||||
p += 4 + sz // skip identifier
|
||||
} else if p < len(b) && b[p] == 0 {
|
||||
p++
|
||||
}
|
||||
if p >= len(b) {
|
||||
return 0, ""
|
||||
}
|
||||
docLen := int(int32(binary.LittleEndian.Uint32(b[p:])))
|
||||
if p+docLen > len(b) {
|
||||
docLen = len(b) - p
|
||||
}
|
||||
doc := b[p : p+docLen]
|
||||
var ok float64
|
||||
var errmsg string
|
||||
i := 4
|
||||
for i < len(doc)-1 {
|
||||
typ := doc[i]
|
||||
i++
|
||||
start := i
|
||||
for i < len(doc) && doc[i] != 0 {
|
||||
i++
|
||||
}
|
||||
name := string(doc[start:i])
|
||||
i++ // skip null
|
||||
switch typ {
|
||||
case 0x01: // double
|
||||
if i+8 <= len(doc) {
|
||||
bits := binary.LittleEndian.Uint64(doc[i:])
|
||||
if name == "ok" {
|
||||
ok = math.Float64frombits(bits)
|
||||
}
|
||||
}
|
||||
i += 8
|
||||
case 0x10: // int32
|
||||
if i+4 <= len(doc) {
|
||||
v := int32(binary.LittleEndian.Uint32(doc[i:]))
|
||||
if name == "ok" {
|
||||
ok = float64(v)
|
||||
}
|
||||
}
|
||||
i += 4
|
||||
case 0x02: // string
|
||||
if i+4 <= len(doc) {
|
||||
n := int(int32(binary.LittleEndian.Uint32(doc[i:])))
|
||||
i += 4
|
||||
if n > 0 && i+n <= len(doc) {
|
||||
if name == "errmsg" {
|
||||
errmsg = string(doc[i : i+n-1])
|
||||
}
|
||||
i += n
|
||||
}
|
||||
}
|
||||
case 0x07: // objectId
|
||||
i += 12
|
||||
case 0x08: // bool
|
||||
i++
|
||||
case 0x03, 0x04: // document/array
|
||||
if i+4 <= len(doc) {
|
||||
n := int(int32(binary.LittleEndian.Uint32(doc[i:])))
|
||||
i += n
|
||||
}
|
||||
case 0x09: // datetime
|
||||
i += 8
|
||||
case 0x0A: // null
|
||||
case 0x11: // timestamp
|
||||
i += 8
|
||||
default:
|
||||
// Unknown type: abandon parsing to avoid mis-stepping.
|
||||
return ok, errmsg
|
||||
}
|
||||
}
|
||||
return ok, errmsg
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package probe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
func listenFake(t *testing.T, handle func(net.Conn)) (string, int) {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
handle(conn)
|
||||
conn.Close()
|
||||
}()
|
||||
host, port := splitAddr(t, ln.Addr().String())
|
||||
return host, port
|
||||
}
|
||||
|
||||
func splitAddr(t *testing.T, addr string) (string, int) {
|
||||
t.Helper()
|
||||
host, portStr, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return host, mustAtoi(portStr)
|
||||
}
|
||||
|
||||
func TestProbeRedisReadable(t *testing.T) {
|
||||
host, port := listenFake(t, func(c net.Conn) {
|
||||
buf := make([]byte, 6)
|
||||
_, _ = io.ReadFull(c, buf)
|
||||
_, _ = c.Write([]byte("+PONG\r\n"))
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, detail, ok := p.probeRedis(context.Background(), host, port)
|
||||
if !ok || state != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s (%s) ok=%v, want public_read", state, detail, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeRedisAuthRequired(t *testing.T) {
|
||||
host, port := listenFake(t, func(c net.Conn) {
|
||||
buf := make([]byte, 6)
|
||||
_, _ = io.ReadFull(c, buf)
|
||||
_, _ = c.Write([]byte("-NOAUTH Authentication required.\r\n"))
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, _, _ := p.probeRedis(context.Background(), host, port)
|
||||
if state != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbePostgresTrust(t *testing.T) {
|
||||
host, port := listenFake(t, func(c net.Conn) {
|
||||
ssl := make([]byte, 8)
|
||||
if _, err := io.ReadFull(c, ssl); err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = c.Write([]byte{'N'})
|
||||
var l [4]byte
|
||||
if _, err := io.ReadFull(c, l[:]); err != nil {
|
||||
return
|
||||
}
|
||||
n := int(binary.BigEndian.Uint32(l[:])) - 4
|
||||
_, _ = io.ReadFull(c, make([]byte, n))
|
||||
// AuthenticationOk: 'R', length 8, auth type 0
|
||||
_, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 0})
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, detail, _ := p.probePostgres(context.Background(), host, port)
|
||||
if state != model.ExposurePublicRead {
|
||||
t.Fatalf("got %s (%s), want public_read", state, detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbePostgresPassword(t *testing.T) {
|
||||
host, port := listenFake(t, func(c net.Conn) {
|
||||
ssl := make([]byte, 8)
|
||||
if _, err := io.ReadFull(c, ssl); err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = c.Write([]byte{'N'})
|
||||
var l [4]byte
|
||||
if _, err := io.ReadFull(c, l[:]); err != nil {
|
||||
return
|
||||
}
|
||||
n := int(binary.BigEndian.Uint32(l[:])) - 4
|
||||
_, _ = io.ReadFull(c, make([]byte, n))
|
||||
// AuthenticationMD5Password: 'R', length 8, auth type 5
|
||||
_, _ = c.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 5})
|
||||
})
|
||||
p := New(Config{Workers: 1, Timeout: time.Second})
|
||||
state, _, _ := p.probePostgres(context.Background(), host, port)
|
||||
if state != model.ExposureAuthRequired {
|
||||
t.Fatalf("got %s, want auth_required", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBSONFields(t *testing.T) {
|
||||
var doc []byte
|
||||
addDouble := func(name string, v uint64) {
|
||||
doc = append(doc, 0x01)
|
||||
doc = append(doc, appendCString(nil, name)...)
|
||||
var tmp [8]byte
|
||||
binary.LittleEndian.PutUint64(tmp[:], v)
|
||||
doc = append(doc, tmp[:]...)
|
||||
}
|
||||
addString := func(name, val string) {
|
||||
doc = append(doc, 0x02)
|
||||
doc = append(doc, appendCString(nil, name)...)
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(len(val)+1))
|
||||
doc = append(doc, tmp[:]...)
|
||||
doc = append(doc, appendCString(nil, val)...)
|
||||
}
|
||||
addDouble("ok", 0x3FF0000000000000) // 1.0
|
||||
addString("errmsg", "requires authentication")
|
||||
body := make([]byte, 0, 5+len(doc)+5)
|
||||
body = append(body, 0, 0, 0, 0, 0) // flags + section kind 0
|
||||
full := make([]byte, 4, 4+len(doc)+1)
|
||||
binary.LittleEndian.PutUint32(full, uint32(4+len(doc)+1))
|
||||
full = append(full, doc...)
|
||||
full = append(full, 0)
|
||||
body = append(body, full...)
|
||||
|
||||
ok, errmsg := parseBSONFields(body)
|
||||
if ok != 1 || errmsg != "requires authentication" {
|
||||
t.Fatalf("got ok=%v errmsg=%q", ok, errmsg)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user