114 lines
4.6 KiB
Python
114 lines
4.6 KiB
Python
"""Strict outbound payload fetcher. HTTP(S) GET only; never executes, pipes or writes files.
|
|
|
|
Egress policy: only http/https on ports 80/443; the host must resolve exclusively to global
|
|
unicast addresses (private/loopback/link-local/metadata rejected); the validated IP is pinned
|
|
for the actual connect so DNS rebinding cannot redirect it; redirects are re-validated and
|
|
depth-limited; size and time are bounded. Downloaded bytes go only to inert evidence storage.
|
|
"""
|
|
import http.client
|
|
import ipaddress
|
|
import re
|
|
import socket
|
|
import ssl
|
|
import threading
|
|
import urllib.parse
|
|
|
|
URL_RE = re.compile(r'https?://[^\s\'"`;|&$()<>\\]+', re.I)
|
|
MAX_BYTES = 16 * 1024 * 1024
|
|
TIMEOUT = 10
|
|
ALLOWED_PORTS = (80, 443)
|
|
|
|
|
|
def extract_urls(text):
|
|
return URL_RE.findall(text or '')
|
|
|
|
|
|
def public_addresses(host):
|
|
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
|
addresses = []
|
|
for info in infos:
|
|
address = info[4][0]
|
|
ip = ipaddress.ip_address(address)
|
|
if not ip.is_global or ip.is_multicast:
|
|
raise ValueError('host resolves to a non-public address')
|
|
if address not in addresses:
|
|
addresses.append(address)
|
|
if not addresses:
|
|
raise ValueError('host did not resolve')
|
|
return addresses
|
|
|
|
|
|
class _PinnedHTTP(http.client.HTTPConnection):
|
|
def __init__(self, host, ip, port):
|
|
super().__init__(host, port, timeout=TIMEOUT)
|
|
self._ip = ip
|
|
|
|
def connect(self):
|
|
self.sock = socket.create_connection((self._ip, self.port), self.timeout)
|
|
|
|
|
|
class _PinnedHTTPS(http.client.HTTPSConnection):
|
|
def __init__(self, host, ip, port):
|
|
super().__init__(host, port, timeout=TIMEOUT, context=ssl.create_default_context())
|
|
self._ip = ip
|
|
|
|
def connect(self):
|
|
raw = socket.create_connection((self._ip, self.port), self.timeout)
|
|
self.sock = self._context.wrap_socket(raw, server_hostname=self.host)
|
|
|
|
|
|
class Fetcher:
|
|
def __init__(self, emit, resolver=public_addresses, max_bytes=MAX_BYTES,
|
|
ports=ALLOWED_PORTS, redirects=3, slots=4, on_failure=None):
|
|
self.emit, self.resolver, self.on_failure = emit, resolver, on_failure
|
|
self.max_bytes, self.ports, self.redirects = max_bytes, ports, redirects
|
|
self.slots = threading.BoundedSemaphore(slots)
|
|
|
|
def submit(self, text, context):
|
|
for url in list(dict.fromkeys(extract_urls(text)))[:4]:
|
|
if not self.slots.acquire(False):
|
|
self._safe_emit('payload_fetch_rejected', url=url, reason='concurrency', **context)
|
|
continue
|
|
threading.Thread(target=self._run, args=(url, context), daemon=True).start()
|
|
|
|
def _safe_emit(self, event, **data):
|
|
try:
|
|
self.emit(event, **data)
|
|
except OSError as error:
|
|
if self.on_failure:
|
|
self.on_failure(error)
|
|
|
|
def _run(self, url, context):
|
|
try:
|
|
self._fetch(url, context)
|
|
except Exception as error:
|
|
self._safe_emit('payload_fetch_error', url=url, error=type(error).__name__, **context)
|
|
finally:
|
|
self.slots.release()
|
|
|
|
def _fetch(self, url, context, depth=0):
|
|
parts = urllib.parse.urlsplit(url)
|
|
if parts.scheme not in ('http', 'https') or not parts.hostname:
|
|
raise ValueError('blocked scheme or host')
|
|
port = parts.port or (443 if parts.scheme == 'https' else 80)
|
|
if port not in self.ports:
|
|
raise ValueError('blocked port')
|
|
addresses = self.resolver(parts.hostname)
|
|
connection = (_PinnedHTTPS if parts.scheme == 'https' else _PinnedHTTP)(
|
|
parts.hostname, addresses[0], port)
|
|
path = urllib.parse.urlunsplit(('', '', parts.path or '/', parts.query, ''))
|
|
try:
|
|
connection.request('GET', path, headers={'Host': parts.hostname,
|
|
'User-Agent': 'CitrixGateway', 'Accept': '*/*', 'Connection': 'close'})
|
|
response = connection.getresponse()
|
|
data = response.read(self.max_bytes + 1)
|
|
status, location = response.status, response.getheader('Location')
|
|
content_type = (response.getheader('Content-Type') or '')[:200]
|
|
finally:
|
|
connection.close()
|
|
self._safe_emit('payload_download', url=url, status=status, content_type=content_type,
|
|
truncated=len(data) > self.max_bytes, content=data[:self.max_bytes], **context)
|
|
# ponytail: only 3 redirect hops; raise the cap if a real payload chain needs more.
|
|
if location and status in (301, 302, 303, 307, 308) and depth < self.redirects:
|
|
self._fetch(urllib.parse.urljoin(url, location), context, depth + 1)
|