discover Azure Blob containers by account name
Azure has two levels: the account maps to DNS (<account>.blob.core.windows.net) and anonymous access is per container with no anonymous container enumeration. Discovery now tests incoming labels as Azure account names (DNS existence) and, for accounts that exist, probes a small set of common containers, indexing any that list anonymously. CONTENT still never downloaded. Also document that docker compose down -v wipes all discovered assets; use stop/start to keep the pgdata volume. Live: connectsystem.blob.core.windows.net indexed (33 objects).
This commit is contained in:
1 parent
dc7bdae99d
commit
32035d0b9d
3 files changed
+111
-4
No files matched your search
@@ -160,6 +160,92 @@ func (d *Discoverer) testName(ctx context.Context, name string) {
|
||||
}
|
||||
d.test(ctx, t)
|
||||
}
|
||||
d.scanAzure(ctx, name)
|
||||
}
|
||||
|
||||
// scanAzure handles Azure's two-level model. The account name maps to DNS
|
||||
// (<account>.blob.core.windows.net), so a nonexistent account fails the first
|
||||
// request and we stop. Anonymous access is per container and Microsoft offers no
|
||||
// anonymous container enumeration, so the container names are the one place a
|
||||
// wordlist is unavoidable.
|
||||
func (d *Discoverer) scanAzure(ctx context.Context, account string) {
|
||||
if !validAzureAccount(account) {
|
||||
return
|
||||
}
|
||||
key := "az:" + account
|
||||
if _, dup := d.seen.Get(key); dup {
|
||||
return
|
||||
}
|
||||
d.seen.Set(key, true, 12*time.Hour)
|
||||
|
||||
host := account + ".blob.core.windows.net"
|
||||
if !d.acquire(ctx) {
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/?comp=list", nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
return // account does not exist (NXDOMAIN) or is unreachable
|
||||
}
|
||||
_, _ = io.CopyN(io.Discard, resp.Body, 512)
|
||||
resp.Body.Close()
|
||||
|
||||
// Account exists. Try known/common containers for anonymous listing.
|
||||
var (
|
||||
entries []model.Entry
|
||||
found []string
|
||||
)
|
||||
for _, container := range index.AzureContainers(account) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if d.cfg.Metrics != nil {
|
||||
d.cfg.Metrics.BucketCandidates.Add(1)
|
||||
}
|
||||
if !d.acquire(ctx) {
|
||||
return
|
||||
}
|
||||
c := model.Candidate{
|
||||
Hostname: host, Original: host, Resource: account, Container: container,
|
||||
Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
|
||||
ResourceSpecific: true, Source: model.SourceDiscovery,
|
||||
}
|
||||
e, err := d.cfg.Indexer.List(ctx, c)
|
||||
if err != nil || len(e) == 0 {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, e...)
|
||||
found = append(found, container)
|
||||
}
|
||||
if len(entries) == 0 || d.cfg.OnOpen == nil {
|
||||
return
|
||||
}
|
||||
if d.cfg.Metrics != nil {
|
||||
d.cfg.Metrics.BucketOpen.Add(1)
|
||||
}
|
||||
c := model.Candidate{
|
||||
Hostname: host, Original: host, Resource: account,
|
||||
Provider: model.ProviderAzure, Service: model.ServiceBlobStorage,
|
||||
ResourceSpecific: true, Source: model.SourceDiscovery,
|
||||
}
|
||||
d.cfg.OnOpen(ctx, c, "azure public container(s): "+strings.Join(found, ","), entries)
|
||||
}
|
||||
|
||||
// validAzureAccount mirrors Azure's naming rules: 3-24 lowercase alphanumerics.
|
||||
func validAzureAccount(s string) bool {
|
||||
if len(s) < 3 || len(s) > 24 {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Target is one provider listing endpoint for a candidate name.
|
||||
|
||||
@@ -17,6 +17,19 @@ func TestCandidatesAreExactLabelsOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidAzureAccount(t *testing.T) {
|
||||
for _, ok := range []string{"myaccount", "abc123", "a1b2c3d4e5f6g7h8i9j0k1l2"} {
|
||||
if !validAzureAccount(ok) {
|
||||
t.Fatalf("%q should be a valid Azure account name", ok)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"ab", "has-hyphen", "UPPER", "with.dot", "a1b2c3d4e5f6g7h8i9j0k1l2m3"} {
|
||||
if validAzureAccount(bad) {
|
||||
t.Fatalf("%q should be rejected as an Azure account name", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBaseLabelsSkipsNoise(t *testing.T) {
|
||||
got := baseLabels("www.example.com")
|
||||
if len(got) != 1 || got[0] != "example" {
|
||||
|
||||
Reference in new issue
Block a user