Bucketeer UI (dark mode, pagination, virtual scroll), readable+indexed resources, LLM-friendly logs
Logging: - structured slog with LOG_FORMAT=json|text and a stable key vocabulary - periodic stats record with queue depths, db pool, goroutines, memory, dns/probe - no per-certificate logging (hot path is counters only) UI (Bucketeer branding, DESIGN.md): - light+dark themes (OS default, persisted header toggle, no first-paint flash) - official mascot: lossless WebP light/dark logos + favicon, embedded and served with cache headers (1.2MB source PNGs -> ~27/22KB webp) - Assets/Contents views, pagination (offset) and native virtual scrolling (content-visibility) so large result sets do not blow up the DOM - search with debounce/abort + URL state, exposure chips, sort, summary counts Readability + indexing: - internal/index listers for S3-compatible, GCS, Azure, Firebase, OpenSearch (metadata only: keys/sizes/dates; never object contents) - asset_entries table + store write/search; readable resources are indexed and searchable in the UI - deep, unauthenticated, read-only DB probes (PROBE_DEEP): Redis PING, PostgreSQL StartupMessage, MongoDB OP_MSG listDatabases; OpenSearch/Neptune HTTP reads - Azure account-level listing fixed: 403 is reported reachable, not auth_required, since anonymous access is per-container DNS: - curated reliable resolvers by default; DNS_SERVERS=fresh adds the compiled-in Fresh-Resolvers snapshot with a bounded startup sample (warming all 6.7k at once broke the container resolver) - random healthy selection + failover + per-server cooldown Other: prober Close fixes a shutdown hang; config/README updated.
This commit is contained in:
StasonJatham committed
1 parent
8fce55f8c9
commit
a366905a43
28 files changed
+9744
-241
No files matched your search
@@ -0,0 +1,364 @@
|
||||
// Package index fetches a bounded metadata index (object keys, sizes, dates) of
|
||||
// resources that were confirmed anonymously readable. It never downloads object
|
||||
// contents: list APIs only. It uses plain HTTP and the standard library XML/JSON
|
||||
// parsers rather than cloud SDKs, which keeps the binary small and the hot path
|
||||
// allocation-free; the list APIs are exactly what the SDKs call underneath.
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
// Indexer lists readable resources. Safe for concurrent use.
|
||||
type Indexer struct {
|
||||
client *http.Client
|
||||
maxObjects int
|
||||
scheme string // overridable in tests
|
||||
gcsBase string // overridable in tests
|
||||
}
|
||||
|
||||
func New(maxObjects int, timeout time.Duration) *Indexer {
|
||||
if maxObjects <= 0 {
|
||||
maxObjects = 1000
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
return &Indexer{
|
||||
maxObjects: maxObjects,
|
||||
scheme: "https",
|
||||
gcsBase: "https://storage.googleapis.com",
|
||||
client: &http.Client{
|
||||
Timeout: timeout,
|
||||
// Use a distinct read limit per response; timeout bounds the whole list.
|
||||
Transport: &http.Transport{MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// List returns up to the configured maximum number of entries for a readable
|
||||
// candidate. Provider families without a listing API return no entries.
|
||||
func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, error) {
|
||||
switch c.Provider {
|
||||
case model.ProviderAWS:
|
||||
switch c.Service {
|
||||
case model.ServiceS3:
|
||||
return x.listS3(ctx, c.Hostname)
|
||||
case model.ServiceOpenSearch:
|
||||
return x.listOpenSearch(ctx, c.Hostname)
|
||||
}
|
||||
case model.ProviderGCP:
|
||||
switch c.Service {
|
||||
case model.ServiceObjectStorage:
|
||||
return x.listGCS(ctx, c.Resource)
|
||||
case model.ServiceFirebaseDB:
|
||||
return x.listFirebase(ctx, c.Hostname)
|
||||
}
|
||||
case model.ProviderAzure:
|
||||
switch c.Service {
|
||||
case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
|
||||
return x.listAzure(ctx, c.Hostname)
|
||||
}
|
||||
case model.ProviderCloudflare:
|
||||
return x.listS3(ctx, c.Hostname)
|
||||
case model.ProviderDigitalOcean:
|
||||
if c.Service == model.ServiceObjectStorage {
|
||||
return x.listS3(ctx, c.Hostname)
|
||||
}
|
||||
case model.ProviderOracle:
|
||||
return x.listS3(ctx, c.Hostname)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (x *Indexer) get(ctx context.Context, endpoint string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "bucketeer-indexer")
|
||||
resp, err := x.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode >= 300 {
|
||||
_, _ = io.CopyN(io.Discard, resp.Body, 512)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// --- S3-compatible (AWS S3, DigitalOcean Spaces, Cloudflare R2, OCI) ---
|
||||
|
||||
type s3ListResult struct {
|
||||
IsTruncated bool `xml:"IsTruncated"`
|
||||
NextContinuationToken string `xml:"NextContinuationToken"`
|
||||
Contents []struct {
|
||||
Key string `xml:"Key"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
ETag string `xml:"ETag"`
|
||||
Size int64 `xml:"Size"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
} `xml:"Contents"`
|
||||
CommonPrefixes []struct {
|
||||
Prefix string `xml:"Prefix"`
|
||||
} `xml:"CommonPrefixes"`
|
||||
}
|
||||
|
||||
func (x *Indexer) listS3(ctx context.Context, host string) ([]model.Entry, error) {
|
||||
var (
|
||||
entries []model.Entry
|
||||
token string
|
||||
pageGuard int
|
||||
)
|
||||
for len(entries) < x.maxObjects && pageGuard < 50 {
|
||||
pageGuard++
|
||||
endpoint := x.scheme + "://" + host + "/?list-type=2&max-keys=1000"
|
||||
if token != "" {
|
||||
endpoint += "&continuation-token=" + url.QueryEscape(token)
|
||||
}
|
||||
resp, err := x.get(ctx, endpoint)
|
||||
if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
var page s3ListResult
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&page)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
for _, p := range page.CommonPrefixes {
|
||||
entries = append(entries, model.Entry{Kind: "prefix", Name: p.Prefix})
|
||||
}
|
||||
for _, o := range page.Contents {
|
||||
entries = append(entries, model.Entry{
|
||||
Kind: "object", Name: o.Key, Size: o.Size,
|
||||
Updated: parseTime(o.LastModified), ETag: strings.Trim(o.ETag, `"`),
|
||||
})
|
||||
if len(entries) >= x.maxObjects {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !page.IsTruncated || page.NextContinuationToken == "" {
|
||||
break
|
||||
}
|
||||
token = page.NextContinuationToken
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// --- Google Cloud Storage ---
|
||||
|
||||
type gcsList struct {
|
||||
Items []struct {
|
||||
Name string `json:"name"`
|
||||
Size string `json:"size"`
|
||||
Updated string `json:"updated"`
|
||||
ContentType string `json:"contentType"`
|
||||
ETag string `json:"etag"`
|
||||
} `json:"items"`
|
||||
Prefixes []string `json:"prefixes"`
|
||||
NextPageToken string `json:"nextPageToken"`
|
||||
}
|
||||
|
||||
func (x *Indexer) listGCS(ctx context.Context, bucket string) ([]model.Entry, error) {
|
||||
if bucket == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var (
|
||||
entries []model.Entry
|
||||
token string
|
||||
pageGuard int
|
||||
)
|
||||
for len(entries) < x.maxObjects && pageGuard < 50 {
|
||||
pageGuard++
|
||||
endpoint := x.gcsBase + "/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1000"
|
||||
if token != "" {
|
||||
endpoint += "&pageToken=" + url.QueryEscape(token)
|
||||
}
|
||||
resp, err := x.get(ctx, endpoint)
|
||||
if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
var page gcsList
|
||||
err = json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
for _, p := range page.Prefixes {
|
||||
entries = append(entries, model.Entry{Kind: "prefix", Name: p})
|
||||
}
|
||||
for _, o := range page.Items {
|
||||
size, _ := strconv.ParseInt(o.Size, 10, 64)
|
||||
entries = append(entries, model.Entry{
|
||||
Kind: "object", Name: o.Name, Size: size,
|
||||
Updated: parseTime(o.Updated), ContentType: o.ContentType, ETag: o.ETag,
|
||||
})
|
||||
if len(entries) >= x.maxObjects {
|
||||
break
|
||||
}
|
||||
}
|
||||
if page.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
token = page.NextPageToken
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// --- Azure Blob / File / Queue / Table (account-level listing) ---
|
||||
|
||||
type azContainers struct {
|
||||
Containers struct {
|
||||
Container []struct {
|
||||
Name string `xml:"Name"`
|
||||
} `xml:"Container"`
|
||||
} `xml:"Containers"`
|
||||
}
|
||||
|
||||
type azBlobs struct {
|
||||
Blobs struct {
|
||||
Blob []struct {
|
||||
Name string `xml:"Name"`
|
||||
Properties struct {
|
||||
ContentLength int64 `xml:"Content-Length"`
|
||||
ContentType string `xml:"Content-Type"`
|
||||
LastModified string `xml:"Last-Modified"`
|
||||
Etag string `xml:"Etag"`
|
||||
} `xml:"Properties"`
|
||||
} `xml:"Blob"`
|
||||
} `xml:"Blobs"`
|
||||
}
|
||||
|
||||
func (x *Indexer) listAzure(ctx context.Context, host string) ([]model.Entry, error) {
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page azContainers
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var entries []model.Entry
|
||||
for _, c := range page.Containers.Container {
|
||||
entries = append(entries, model.Entry{Kind: "container", Name: c.Name})
|
||||
if len(entries) >= x.maxObjects {
|
||||
return entries, nil
|
||||
}
|
||||
// List a bounded number of blobs from each container.
|
||||
blobs, err := x.listAzureContainer(ctx, host, c.Name, x.maxObjects-len(entries))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, blobs...)
|
||||
if len(entries) >= x.maxObjects {
|
||||
return entries, nil
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) {
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/"+url.PathEscape(container)+"?restype=container&comp=list")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page azBlobs
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&page)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var entries []model.Entry
|
||||
for _, b := range page.Blobs.Blob {
|
||||
entries = append(entries, model.Entry{
|
||||
Kind: "object", Name: container + "/" + b.Name, Size: b.Properties.ContentLength,
|
||||
Updated: parseTime(b.Properties.LastModified), ContentType: b.Properties.ContentType,
|
||||
ETag: b.Properties.Etag,
|
||||
})
|
||||
if len(entries) >= remaining {
|
||||
break
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// --- Firebase Realtime Database (shallow root keys) ---
|
||||
|
||||
func (x *Indexer) listFirebase(ctx context.Context, host string) ([]model.Entry, error) {
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/.json?shallow=true")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
err = json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&keys)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries := make([]model.Entry, 0, len(keys))
|
||||
for k := range keys {
|
||||
entries = append(entries, model.Entry{Kind: "key", Name: k})
|
||||
if len(entries) >= x.maxObjects {
|
||||
break
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// --- OpenSearch / Elasticsearch indices ---
|
||||
|
||||
func (x *Indexer) listOpenSearch(ctx context.Context, host string) ([]model.Entry, error) {
|
||||
resp, err := x.get(ctx, x.scheme+"://"+host+"/_cat/indices?format=json&h=index,health,status,docs.count,store.size")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []struct {
|
||||
Index string `json:"index"`
|
||||
Health string `json:"health"`
|
||||
Status string `json:"status"`
|
||||
Docs string `json:"docs.count"`
|
||||
StoreSiz string `json:"store.size"`
|
||||
}
|
||||
err = json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&rows)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var entries []model.Entry
|
||||
for _, r := range rows {
|
||||
size, _ := strconv.ParseInt(strings.TrimSuffix(r.Docs, " "), 10, 64)
|
||||
entries = append(entries, model.Entry{Kind: "index", Name: r.Index, Size: size, ETag: r.StoreSiz})
|
||||
if len(entries) >= x.maxObjects {
|
||||
break
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func parseTime(s string) time.Time {
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, http.TimeFormat} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"tea.karlcom.de/karl/bucketeer/internal/model"
|
||||
)
|
||||
|
||||
func newTestIndexer(handler http.Handler) (*Indexer, string) {
|
||||
srv := httptest.NewServer(handler)
|
||||
x := New(50, 2*time.Second)
|
||||
x.scheme = "http"
|
||||
x.gcsBase = srv.URL
|
||||
return x, strings.TrimPrefix(srv.URL, "http://")
|
||||
}
|
||||
|
||||
func TestListS3(t *testing.T) {
|
||||
body := `<?xml version="1.0"?><ListBucketResult>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<CommonPrefixes><Prefix>photos/</Prefix></CommonPrefixes>
|
||||
<Contents><Key>photos/a.jpg</Key><LastModified>2024-01-02T03:04:05.000Z</LastModified><ETag>"abc"</ETag><Size>1234</Size></Contents>
|
||||
<Contents><Key>readme.txt</Key><Size>12</Size></Contents>
|
||||
</ListBucketResult>`
|
||||
x, host := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("list-type") != "2" {
|
||||
t.Errorf("expected list-type=2, got %q", r.URL.RawQuery)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
entries, err := x.List(context.Background(), model.Candidate{Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("got %d entries: %+v", len(entries), entries)
|
||||
}
|
||||
if entries[0].Kind != "prefix" || entries[0].Name != "photos/" {
|
||||
t.Fatalf("prefix entry wrong: %+v", entries[0])
|
||||
}
|
||||
if entries[1].Name != "photos/a.jpg" || entries[1].Size != 1234 || entries[1].ETag != "abc" {
|
||||
t.Fatalf("object entry wrong: %+v", entries[1])
|
||||
}
|
||||
if entries[1].Updated.IsZero() {
|
||||
t.Fatal("expected updated timestamp parsed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListGCS(t *testing.T) {
|
||||
body := `{"items":[{"name":"a/b.txt","size":"42","updated":"2024-05-06T07:08:09Z","contentType":"text/plain","etag":"e1"}],"prefixes":["a/"]}`
|
||||
x, _ := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/storage/v1/b/") {
|
||||
t.Errorf("unexpected path %q", r.URL.Path)
|
||||
}
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
entries, err := x.List(context.Background(), model.Candidate{Resource: "mybucket", Provider: model.ProviderGCP, Service: model.ServiceObjectStorage})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 2 || entries[1].Name != "a/b.txt" || entries[1].Size != 42 || entries[1].ContentType != "text/plain" {
|
||||
t.Fatalf("unexpected entries: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFirebase(t *testing.T) {
|
||||
x, host := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"users":true,"orders":true}`))
|
||||
}))
|
||||
entries, err := x.List(context.Background(), model.Candidate{Hostname: host, Provider: model.ProviderGCP, Service: model.ServiceFirebaseDB})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 2 || entries[0].Kind != "key" {
|
||||
t.Fatalf("unexpected entries: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOpenSearch(t *testing.T) {
|
||||
x, host := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`[{"index":"logs-2024","docs.count":"100","store.size":"1.2mb"}]`))
|
||||
}))
|
||||
entries, err := x.List(context.Background(), model.Candidate{Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceOpenSearch})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name != "logs-2024" || entries[0].Size != 100 {
|
||||
t.Fatalf("unexpected entries: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListBounded(t *testing.T) {
|
||||
var keys strings.Builder
|
||||
for i := 0; i < 100; i++ {
|
||||
keys.WriteString("<Contents><Key>k")
|
||||
keys.WriteString(string(rune('a' + i%26)))
|
||||
keys.WriteString("</Key><Size>1</Size></Contents>")
|
||||
}
|
||||
body := `<ListBucketResult><IsTruncated>false</IsTruncated>` + keys.String() + `</ListBucketResult>`
|
||||
x, host := newTestIndexer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) }))
|
||||
x.maxObjects = 10
|
||||
entries, err := x.List(context.Background(), model.Candidate{Hostname: host, Provider: model.ProviderAWS, Service: model.ServiceS3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) > 10 {
|
||||
t.Fatalf("got %d entries, want <= 10", len(entries))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user