bucketeer/internal/index/index.go

// Package index fetches a bounded metadata index (object keys, sizes, dates) of
// resources that were confirmed anonymously readable. It never downloads object
// contents: list APIs only. It uses plain HTTP and the standard library XML/JSON
// parsers rather than cloud SDKs, which keeps the binary small and the hot path
// allocation-free; the list APIs are exactly what the SDKs call underneath.
package index

import (
	"context"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"tea.karlcom.de/karl/bucketeer/internal/model"
)

// Indexer lists readable resources. Safe for concurrent use.
type Indexer struct {
	client     *http.Client
	maxObjects int
	scheme     string // overridable in tests
	gcsBase    string // overridable in tests
}

func New(maxObjects int, timeout time.Duration) *Indexer {
	if maxObjects <= 0 {
		maxObjects = 1000
	}
	if timeout <= 0 {
		timeout = 10 * time.Second
	}
	return &Indexer{
		maxObjects: maxObjects,
		scheme:     "https",
		gcsBase:    "https://storage.googleapis.com",
		client: &http.Client{
			Timeout: timeout,
			// Use a distinct read limit per response; timeout bounds the whole list.
			Transport: &http.Transport{MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second},
		},
	}
}

// List returns up to the configured maximum number of entries for a readable
// candidate. Provider families without a listing API return no entries.
func (x *Indexer) List(ctx context.Context, c model.Candidate) ([]model.Entry, error) {
	switch c.Provider {
	case model.ProviderAWS:
		switch c.Service {
		case model.ServiceS3:
			return x.listS3(ctx, c.Hostname)
		}
	case model.ProviderGCP:
		switch c.Service {
		case model.ServiceObjectStorage:
			return x.listGCS(ctx, c.Resource)
		case model.ServiceFirebaseDB:
			return x.listFirebase(ctx, c.Hostname)
		}
	case model.ProviderAzure:
		switch c.Service {
		case model.ServiceBlobStorage, model.ServiceFileStorage, model.ServiceQueue, model.ServiceTableStorage:
			return x.listAzure(ctx, c.Hostname, c.Resource, c.Container)
		}
	case model.ProviderCloudflare:
		// R2 public URLs serve objects but never allow anonymous listing.
		if strings.HasSuffix(c.Hostname, ".r2.dev") || strings.HasSuffix(c.Hostname, ".cloudflarestorage.com") {
			return nil, nil
		}
		return x.listS3(ctx, c.Hostname)
	case model.ProviderDigitalOcean:
		if c.Service == model.ServiceObjectStorage {
			return x.listS3(ctx, c.Hostname)
		}
	case model.ProviderOracle:
		return x.listS3(ctx, c.Hostname)
	}
	return nil, nil
}

func (x *Indexer) get(ctx context.Context, endpoint string) (*http.Response, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "bucketeer-indexer")
	resp, err := x.client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode >= 300 {
		_, _ = io.CopyN(io.Discard, resp.Body, 512)
		resp.Body.Close()
		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
	}
	return resp, nil
}

// --- S3-compatible (AWS S3, DigitalOcean Spaces, Cloudflare R2, OCI) ---

type s3ListResult struct {
	IsTruncated           bool   `xml:"IsTruncated"`
	NextContinuationToken string `xml:"NextContinuationToken"`
	Contents              []struct {
		Key          string `xml:"Key"`
		LastModified string `xml:"LastModified"`
		ETag         string `xml:"ETag"`
		Size         int64  `xml:"Size"`
		StorageClass string `xml:"StorageClass"`
	} `xml:"Contents"`
	CommonPrefixes []struct {
		Prefix string `xml:"Prefix"`
	} `xml:"CommonPrefixes"`
}

func (x *Indexer) listS3(ctx context.Context, host string) ([]model.Entry, error) {
	var (
		entries   []model.Entry
		token     string
		pageGuard int
	)
	for len(entries) < x.maxObjects && pageGuard < 50 {
		pageGuard++
		endpoint := x.scheme + "://" + host + "/?list-type=2&max-keys=1000"
		if token != "" {
			endpoint += "&continuation-token=" + url.QueryEscape(token)
		}
		resp, err := x.get(ctx, endpoint)
		if err != nil {
			return entries, err
		}
		var page s3ListResult
		err = xml.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&page)
		resp.Body.Close()
		if err != nil {
			return entries, err
		}
		for _, p := range page.CommonPrefixes {
			entries = append(entries, model.Entry{Kind: "prefix", Name: p.Prefix})
		}
		for _, o := range page.Contents {
			entries = append(entries, model.Entry{
				Kind: "object", Name: o.Key, Size: o.Size,
				Updated: parseTime(o.LastModified), ETag: strings.Trim(o.ETag, `"`),
			})
			if len(entries) >= x.maxObjects {
				break
			}
		}
		if !page.IsTruncated || page.NextContinuationToken == "" {
			break
		}
		token = page.NextContinuationToken
	}
	return entries, nil
}

// --- Google Cloud Storage ---

type gcsList struct {
	Items []struct {
		Name        string `json:"name"`
		Size        string `json:"size"`
		Updated     string `json:"updated"`
		ContentType string `json:"contentType"`
		ETag        string `json:"etag"`
	} `json:"items"`
	Prefixes      []string `json:"prefixes"`
	NextPageToken string   `json:"nextPageToken"`
}

func (x *Indexer) listGCS(ctx context.Context, bucket string) ([]model.Entry, error) {
	if bucket == "" {
		return nil, nil
	}
	var (
		entries   []model.Entry
		token     string
		pageGuard int
	)
	for len(entries) < x.maxObjects && pageGuard < 50 {
		pageGuard++
		endpoint := x.gcsBase + "/storage/v1/b/" + url.PathEscape(bucket) + "/o?maxResults=1000"
		if token != "" {
			endpoint += "&pageToken=" + url.QueryEscape(token)
		}
		resp, err := x.get(ctx, endpoint)
		if err != nil {
			return entries, err
		}
		var page gcsList
		err = json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
		resp.Body.Close()
		if err != nil {
			return entries, err
		}
		for _, p := range page.Prefixes {
			entries = append(entries, model.Entry{Kind: "prefix", Name: p})
		}
		for _, o := range page.Items {
			size, _ := strconv.ParseInt(o.Size, 10, 64)
			entries = append(entries, model.Entry{
				Kind: "object", Name: o.Name, Size: size,
				Updated: parseTime(o.Updated), ContentType: o.ContentType, ETag: o.ETag,
			})
			if len(entries) >= x.maxObjects {
				break
			}
		}
		if page.NextPageToken == "" {
			break
		}
		token = page.NextPageToken
	}
	return entries, nil
}

// --- Azure Blob / File / Queue / Table (account-level listing) ---

type azContainers struct {
	Containers struct {
		Container []struct {
			Name string `xml:"Name"`
		} `xml:"Container"`
	} `xml:"Containers"`
}

type azBlobs struct {
	Blobs struct {
		Blob []struct {
			Name       string `xml:"Name"`
			Properties struct {
				ContentLength int64  `xml:"Content-Length"`
				ContentType   string `xml:"Content-Type"`
				LastModified  string `xml:"Last-Modified"`
				Etag          string `xml:"Etag"`
			} `xml:"Properties"`
		} `xml:"Blob"`
	} `xml:"Blobs"`
}

// AzureContainers returns candidate container names to probe. Anonymous access
// in Azure is configured per container and the container name is not part of the
// hostname, so discovery needs a small dictionary plus account-derived guesses.
// This is a deliberate, bounded heuristic.
func AzureContainers(account string) []string {
	common := []string{
		"$web", "images", "img", "static", "assets", "media", "public", "www", "web",
		"files", "uploads", "data", "content", "docs", "downloads", "resources",
		"backup", "logs", "blog", "site", "app", "store", "photos", "videos",
	}
	seen := map[string]bool{}
	out := make([]string, 0, len(common)+4)
	add := func(s string) {
		if s != "" && !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	for _, c := range common {
		add(c)
	}
	if account != "" {
		add(account)
		for _, suffix := range []string{"dev", "prod", "test", "staging", "storage", "st", "sa"} {
			if strings.HasSuffix(account, suffix) && len(account) > len(suffix) {
				add(strings.TrimSuffix(account, suffix))
			}
		}
	}
	return out
}

func (x *Indexer) listAzure(ctx context.Context, host, account, hint string) ([]model.Entry, error) {
	containers := []string{}
	if hint != "" {
		containers = []string{hint} // container known from a URL path
	}
	if len(containers) == 0 {
		if resp, err := x.get(ctx, x.scheme+"://"+host+"/?comp=list"); err == nil {
			var page azContainers
			if xml.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&page) == nil {
				for _, c := range page.Containers.Container {
					containers = append(containers, c.Name)
				}
			}
			resp.Body.Close()
		}
	}
	if len(containers) == 0 {
		containers = AzureContainers(account)
	}
	var entries []model.Entry
	for _, name := range containers {
		blobs, err := x.listAzureContainer(ctx, host, name, x.maxObjects-len(entries))
		if err != nil {
			continue
		}
		entries = append(entries, blobs...)
		if len(entries) >= x.maxObjects {
			break
		}
	}
	return entries, nil
}

func (x *Indexer) listAzureContainer(ctx context.Context, host, container string, remaining int) ([]model.Entry, error) {
	resp, err := x.get(ctx, x.scheme+"://"+host+"/"+container+"?restype=container&comp=list")
	if err != nil {
		return nil, err
	}
	var page azBlobs
	err = xml.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&page)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	var entries []model.Entry
	for _, b := range page.Blobs.Blob {
		entries = append(entries, model.Entry{
			Kind: "object", Name: container + "/" + b.Name, Size: b.Properties.ContentLength,
			Updated: parseTime(b.Properties.LastModified), ContentType: b.Properties.ContentType,
			ETag: b.Properties.Etag,
		})
		if len(entries) >= remaining {
			break
		}
	}
	return entries, nil
}

// --- Firebase Realtime Database (shallow root keys) ---

func (x *Indexer) listFirebase(ctx context.Context, host string) ([]model.Entry, error) {
	resp, err := x.get(ctx, x.scheme+"://"+host+"/.json?shallow=true")
	if err != nil {
		return nil, err
	}
	var keys map[string]json.RawMessage
	err = json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&keys)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	entries := make([]model.Entry, 0, len(keys))
	for k := range keys {
		entries = append(entries, model.Entry{Kind: "key", Name: k})
		if len(entries) >= x.maxObjects {
			break
		}
	}
	return entries, nil
}

func parseTime(s string) time.Time {
	if s == "" {
		return time.Time{}
	}
	for _, layout := range []string{time.RFC3339Nano, time.RFC3339, http.TimeFormat} {
		if t, err := time.Parse(layout, s); err == nil {
			return t
		}
	}
	return time.Time{}
}