provider capability model, on-demand scan/backfill, more providers
Readability capabilities (exists/object_readable/listable) are now tracked
separately instead of a single exposure flag, matching how providers differ:
- Azure 'Blob' mode = object readable, not listable; 'Container' = listable
- Cloudflare R2 public URLs are never anonymously listable (root 500); R2 is
object-read only and is no longer probed for listing
- S3/GCS: 200 list = listable; 403 + readable index.html/robots.txt =
object readable only
New columns: cloud_assets.listable, cloud_assets.object_readable.
Azure: account-level container listing is never anonymous; discover containers
by probing a bounded common-name set (or the container from a URL path), then
list blobs.
Classifier: added the legacy AWS '.s3-<region>.amazonaws.com' bucket form.
On-demand scan: POST /api/scan {target} classifies a host or URL, follows CNAME
for custom domains, probes, and indexes readable resources; Azure container
comes from the URL path. UI has a Scan box.
Backfill: POST /api/reprobe re-probes already-discovered assets with the current
checks and indexes any readable ones, recovering findings from earlier probes.
DNS: CNAME-only enrichment (A/AAAA resolved only for matches), 64 workers,
larger queue.
Verified live against real public resources:
veteran1.nyc3.digitaloceanspaces.com listable, entries
bucket-01.s3-eu-west-1.amazonaws.com listable, entries
tartugeo-podcast.storage.googleapis.com listable, entries
eroticfeeldev.blob.core.windows.net/images listable, entries
This commit is contained in:
1 parent
a366905a43
commit
62a7d2eb2e
23 files changed
+936
-165
No files matched your search
@@ -88,6 +88,75 @@ type Enricher struct {
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// ResolveCNAME follows just the CNAME chain (no A/AAAA), for the high-volume
|
||||
// enrichment path where only the targets are classified. Cached separately from
|
||||
// full results.
|
||||
func (e *Enricher) ResolveCNAME(ctx context.Context, host string) ([]string, bool) {
|
||||
if e.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, e.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
key := "c:" + host
|
||||
if e.Cache != nil {
|
||||
if res, ok := e.Cache.Get(key); ok {
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.DNSCacheHits.Add(1)
|
||||
}
|
||||
return res.Chain, true
|
||||
}
|
||||
}
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.DNSQueries.Add(1)
|
||||
}
|
||||
chain, err := Follow(ctx, e.Resolver, host, e.MaxDepth)
|
||||
if err != nil && len(chain) == 1 {
|
||||
if e.Metrics != nil && !definitive(err) {
|
||||
e.Metrics.DNSErrors.Add(1)
|
||||
}
|
||||
if e.Cache != nil {
|
||||
e.Cache.Set(key, Result{}, e.NegativeTTL)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
if e.Cache != nil {
|
||||
e.Cache.Set(key, Result{Chain: chain}, e.PositiveTTL)
|
||||
}
|
||||
return chain, true
|
||||
}
|
||||
|
||||
// ResolveIPs resolves A/AAAA for a host, cached.
|
||||
func (e *Enricher) ResolveIPs(ctx context.Context, host string) ([]net.IP, bool) {
|
||||
if e.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, e.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
key := "ip:" + host
|
||||
if e.Cache != nil {
|
||||
if res, ok := e.Cache.Get(key); ok {
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.DNSCacheHits.Add(1)
|
||||
}
|
||||
return res.IPs, true
|
||||
}
|
||||
}
|
||||
ips, err := e.Resolver.LookupIP(ctx, host)
|
||||
if err != nil || len(ips) == 0 {
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.DNSErrors.Add(1)
|
||||
}
|
||||
if e.Cache != nil {
|
||||
e.Cache.Set(key, Result{}, e.NegativeTTL)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
if e.Cache != nil {
|
||||
e.Cache.Set(key, Result{IPs: ips}, e.PositiveTTL)
|
||||
}
|
||||
return ips, true
|
||||
}
|
||||
|
||||
// Resolve returns the result for host. ok is false when the lookup failed;
|
||||
// failures (including "no such host") are negatively cached for NegativeTTL.
|
||||
func (e *Enricher) Resolve(ctx context.Context, host string) (Result, bool) {
|
||||
|
||||
Reference in new issue
Block a user