// Package index fetches a bounded metadata index (object keys, sizes, dates) of // resources that were confirmed anonymously readable. It never downloads object // contents: list APIs only. It uses plain HTTP and the standard library XML/JSON // parsers rather than cloud SDKs, which keeps the binary small and the hot path // allocation-free; the list APIs are exactly what the SDKs call underneath. package index import ( "context" "encoding/json" "encoding/xml" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "tea.karlcom.de/karl/bucketeer/internal/model" ) // Indexer lists readable resources. Safe for concurrent use. type Indexer struct { client *http.Client maxObjects int scheme string // overridable in tests gcsBase string // overridable in tests } func New(maxObjects int, timeout time.Duration) *Indexer { if maxObjects <= 0 { maxObjects = 1000 } if timeout <= 0 { timeout = 10 * time.Second } return &Indexer{ maxObjects: maxObjects, scheme: "https", gcsBase: "https://storage.googleapis.com", client: &http.Client{ Timeout: timeout, // Use a distinct read limit per response; timeout bounds the whole list. Transport: &http.Transport{MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second}, }, } } // List returns up to the configured maximum number of entries for a readable // candidate. Provider families without a listing API return no entries. func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, error) { switch c.Provider { case model.ProviderAWS: switch c.Service { case model.ServiceS3: return x.listS3(ctx, c.Hostname) case model.ServiceOpenSearch: return x.listOpenSearch(ctx, c.Hostname) } case model.ProviderGCP: switch c.Service { case model.ServiceObjectStorage: return x.listGCS(ctx, c.Resource) case model.ServiceFirebaseDB: return x.listFirebase(ctx, c.Hostname) } case model.ProviderAzure: switch c.Service { case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage: return x.listAzure(ctx, c.Hostname, c.Resource, c.Container) } case model.ProviderCloudflare: // R2 public URLs serve objects but never allow anonymous listing. if strings.HasSuffix(c.Hostname, ".r2.dev") || strings.HasSuffix(c.Hostname, ".cloudflarestorage.com") { return nil, nil } return x.listS3(ctx, c.Hostname) case model.ProviderDigitalOcean: if c.Service == model.ServiceObjectStorage { return x.listS3(ctx, c.Hostname) } case model.ProviderOracle: return x.listS3(ctx, c.Hostname) } return nil, nil } func (x *Indexer) get(ctx context.Context, endpoint string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "bucketeer-indexer") resp, err := x.client.Do(req) if err != nil { return nil, err } if resp.StatusCode >= 300 { _, _ = io.CopyN(io.Discard, resp.Body, 512) resp.Body.Close() return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } return resp, nil } // --- S3-compatible (AWS S3, DigitalOcean Spaces, Cloudflare R2, OCI) --- type s3ListResult struct { IsTruncated bool `xml:"IsTruncated"` NextContinuationToken string `xml:"NextContinuationToken"` Contents []struct { Key string `xml:"Key"` LastModified string `xml:"LastModified"` ETag string `xml:"ETag"` Size int64 `xml:"Size"` StorageClass string `xml:"StorageClass"` } `xml:"Contents"` CommonPrefixes []struct { Prefix string `xml:"Prefix"` } `xml:"CommonPrefixes"` } func (x *Indexer) listS3(ctx context.Context, host string) ([]model.Entry, error) { var ( entries []model.Entry token string pageGuard int ) for len(entries) < x.maxObjects && pageGuard < 50 { pageGuard++ endpoint := x.scheme + "://" + host + "/?list-type=2&max-keys=1000" if token != "" { endpoint += "&continuation-token=" + url.QueryEscape(token) } resp, err := x.get(ctx, endpoint) if err != nil { return entries, err } var page s3ListResult err = xml.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&page) resp.Body.Close() if err != nil { return entries, err } for _, p := range page.CommonPrefixes { entries = append(entries, model.Entry{Kind: "prefix", Name: p.Prefix}) } for _, o := range page.Contents { entries = append(entries, model.Entry{ Kind: "object", Name: o.Key, Size: o.Size, Updated: parseTime(o.LastModified), ETag: strings.Trim(o.ETag, `"`), }) if len(entries) >= x.maxObjects { break } } if !page.IsTruncated || page.NextContinuationToken == "" { break } token = page.NextContinuationToken } return entries, nil } // --- Google Cloud Storage --- type gcsList struct { Items []struct { Name string `json:"name"` Size string `json:"size"` Updated string `json:"updated"` ContentType string `json:"contentType"` ETag string `json:"etag"` } `json:"items"` Prefixes []string `json:"prefixes"` NextPageToken string `json:"nextPageToken"` } func (x *Indexer) listGCS(ctx context.Context, bucket string) ([]model.Entry, error) { if bucket == "" { return nil, nil } var ( entries []model.Entry token string pageGuard int ) for len(entries) < x.maxObjects && pageGuard < 50 { pageGuard++ endpoint := x.gcsBase + "/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1000" if token != "" { endpoint += "&pageToken=" + url.QueryEscape(token) } resp, err := x.get(ctx, endpoint) if err != nil { return entries, err } var page gcsList err = json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page) resp.Body.Close() if err != nil { return entries, err } for _, p := range page.Prefixes { entries = append(entries, model.Entry{Kind: "prefix", Name: p}) } for _, o := range page.Items { size, _ := strconv.ParseInt(o.Size, 10, 64) entries = append(entries, model.Entry{ Kind: "object", Name: o.Name, Size: size, Updated: parseTime(o.Updated), ContentType: o.ContentType, ETag: o.ETag, }) if len(entries) >= x.maxObjects { break } } if page.NextPageToken == "" { break } token = page.NextPageToken } return entries, nil } // --- Azure Blob / File / Queue / Table (account-level listing) --- type azContainers struct { Containers struct { Container []struct { Name string `xml:"Name"` } `xml:"Container"` } `xml:"Containers"` } type azBlobs struct { Blobs struct { Blob []struct { Name string `xml:"Name"` Properties struct { ContentLength int64 `xml:"Content-Length"` ContentType string `xml:"Content-Type"` LastModified string `xml:"Last-Modified"` Etag string `xml:"Etag"` } `xml:"Properties"` } `xml:"Blob"` } `xml:"Blobs"` } // AzureContainers returns candidate container names to probe. Anonymous access // in Azure is configured per container and the container name is not part of the // hostname, so discovery needs a small dictionary plus account-derived guesses. // This is a deliberate, bounded heuristic. func AzureContainers(account string) []string { common := []string{ "$web", "images", "img", "static", "assets", "media", "public", "www", "web", "files", "uploads", "data", "content", "docs", "downloads", "resources", "backup", "logs", "blog", "site", "app", "store", "photos", "videos", } seen := map[string]bool{} out := make([]string, 0, len(common)+4) add := func(s string) { if s != "" && !seen[s] { seen[s] = true out = append(out, s) } } for _, c := range common { add(c) } if account != "" { add(account) for _, suffix := range []string{"dev", "prod", "test", "staging", "storage", "st", "sa"} { if strings.HasSuffix(account, suffix) && len(account) > len(suffix) { add(strings.TrimSuffix(account, suffix)) } } } return out } func (x *Indexer) listAzure(ctx context.Context, host, account, hint string) ([]model.Entry, error) { containers := []string{} if hint != "" { containers = []string{hint} // container known from a URL path } if len(containers) == 0 { if resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list"); err == nil { var page azContainers if xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) == nil { for _, c := range page.Containers.Container { containers = append(containers, c.Name) } } resp.Body.Close() } } if len(containers) == 0 { containers = AzureContainers(account) } var entries []model.Entry for _, name := range containers { blobs, err := x.listAzureContainer(ctx, host, name, x.maxObjects-len(entries)) if err != nil { continue } entries = append(entries, blobs...) if len(entries) >= x.maxObjects { break } } return entries, nil } func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) { resp, err := x.get(ctx, x.scheme+"://"+host+"/"+container+"?restype=container&comp=list") if err != nil { return nil, err } var page azBlobs err = xml.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&page) resp.Body.Close() if err != nil { return nil, err } var entries []model.Entry for _, b := range page.Blobs.Blob { entries = append(entries, model.Entry{ Kind: "object", Name: container + "/" + b.Name, Size: b.Properties.ContentLength, Updated: parseTime(b.Properties.LastModified), ContentType: b.Properties.ContentType, ETag: b.Properties.Etag, }) if len(entries) >= remaining { break } } return entries, nil } // --- Firebase Realtime Database (shallow root keys) --- func (x *Indexer) listFirebase(ctx context.Context, host string) ([]model.Entry, error) { resp, err := x.get(ctx, x.scheme+"://"+host+"/.json?shallow=true") if err != nil { return nil, err } var keys map[string]json.RawMessage err = json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&keys) resp.Body.Close() if err != nil { return nil, err } entries := make([]model.Entry, 0, len(keys)) for k := range keys { entries = append(entries, model.Entry{Kind: "key", Name: k}) if len(entries) >= x.maxObjects { break } } return entries, nil } // --- OpenSearch / Elasticsearch indices --- func (x *Indexer) listOpenSearch(ctx context.Context, host string) ([]model.Entry, error) { resp, err := x.get(ctx, x.scheme+"://"+host+"/_cat/indices?format=json&h=index,health,status,docs.count,store.size") if err != nil { return nil, err } var rows []struct { Index string `json:"index"` Health string `json:"health"` Status string `json:"status"` Docs string `json:"docs.count"` StoreSiz string `json:"store.size"` } err = json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&rows) resp.Body.Close() if err != nil { return nil, err } var entries []model.Entry for _, r := range rows { size, _ := strconv.ParseInt(strings.TrimSuffix(r.Docs, " "), 10, 64) entries = append(entries, model.Entry{Kind: "index", Name: r.Index, Size: size, ETag: r.StoreSiz}) if len(entries) >= x.maxObjects { break } } return entries, nil } func parseTime(s string) time.Time { if s == "" { return time.Time{} } for _, layout := range []string{time.RFC3339Nano, time.RFC3339, http.TimeFormat} { if t, err := time.Parse(layout, s); err == nil { return t } } return time.Time{} }