CitrixPot: NetScaler / Citrix ADC Gateway honeypot (initial public release)

deploy/reset-execution.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+"""Hourly VM223 reset, preserving its full disk and pcap first. Never prunes evidence.
+
+--check is read-only. --run stops intake and VM223, archives disk/pcap, validates,
+then rolls back decoy-clean. On any failure it leaves VM223/intake stopped and
+sets the host STOP latch. RAM is not preserved; collector evidence is untouched.
+"""
+import argparse
+import datetime
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import subprocess
+import time
+import uuid
+
+ROOT = Path('/var/lib/vz/honeypot')
+ARCHIVES = ROOT / 'archives'
+DISK = Path('/dev/pve/vm-223-disk-0')
+LIMIT = 16 * 1024**3
+RESERVE = 4 * 1024**3
+
+
+def command(*args, timeout=60):
+    return subprocess.run(args, check=True, capture_output=True, text=True,
+                          timeout=timeout).stdout
+
+
+def qm(*args, timeout=60):
+    return command('/usr/sbin/qm', *args, timeout=timeout)
+
+
+def guest(script):
+    result = json.loads(qm('guest', 'exec', '222', '--timeout', '20', '--',
+                           '/bin/sh', '-c', script, timeout=25))
+    if not result.get('exited') or result.get('exitcode') != 0:
+        raise RuntimeError('collector guest operation failed')
+    return result.get('out-data', '')
+
+
+def sync_dir(path):
+    fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def write_file(path, data):
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    with os.fdopen(fd, 'wb') as stream:
+        stream.write(data)
+        stream.flush()
+        os.fsync(stream.fileno())
+    sync_dir(path.parent)
+
+
+def bounds(min_free=RESERVE):
+    used = sum(p.stat().st_size for p in ARCHIVES.rglob('*') if p.is_file()) if ARCHIVES.exists() else 0
+    if used >= LIMIT:
+        raise RuntimeError('archive 16GiB budget reached; evidence retained')
+    if shutil.disk_usage(ROOT).free < min_free:
+        raise RuntimeError('host free-space reserve reached; evidence retained')
+
+
+def valid_config(config):
+    values = dict(line.split(': ', 1) for line in config.splitlines() if ': ' in line)
+    if (values.get('name') != 'decoy-execution' or
+            not values.get('scsi0', '').startswith('local-lvm:vm-223-disk-0,') or
+            'restrict=on' not in values.get('args', '') or
+            'file=/var/lib/vz/honeypot/execution.pcap' not in values.get('args', '') or
+            any(k.startswith('net') and k[3:].isdigit() for k in values)):
+        raise RuntimeError('VM223 identity or isolation mismatch')
+
+
+def validate():
+    if os.geteuid() != 0:
+        raise RuntimeError('run on designated Proxmox host as root')
+    if ROOT.is_symlink() or not ROOT.is_dir() or (ARCHIVES.exists() and ARCHIVES.is_symlink()):
+        raise RuntimeError('invalid evidence directory')
+    if (ROOT / 'STOP').exists():
+        raise RuntimeError('preexisting host STOP latch requires operator review')
+    if not stat.S_ISBLK(DISK.stat().st_mode):
+        raise RuntimeError('execution disk is not a block device')
+    current = qm('config', '223')
+    baseline = qm('config', '223', '--snapshot', 'decoy-clean')
+    valid_config(current)
+    valid_config(baseline)
+    guest('test -f /etc/decoy-collector && test ! -e /var/lib/decoy/STOP')
+    return current, baseline
+
+
+def digest(path):
+    hashed = hashlib.sha256()
+    with path.open('rb') as stream:
+        while chunk := stream.read(1024**2):
+            bounds()
+            hashed.update(chunk)
+        os.fsync(stream.fileno())
+    return hashed.hexdigest()
+
+
+def install_admin_ca():
+    public = ROOT / 'admin-ca.authorized'
+    if not public.exists():
+        return
+    line = public.read_text().strip()
+    if not line.startswith('cert-authority,principals="decoy-admin",restrict,pty ssh-ed25519 ') or len(line) > 4096 or '\n' in line:
+        raise RuntimeError('invalid dedicated admin CA authorization')
+    script = ("import pathlib,os; p=pathlib.Path('/root/.ssh/authorized_keys'); "
+              "p.parent.mkdir(mode=0o700,exist_ok=True); "
+              "lines=p.read_text().splitlines() if p.exists() else []; "
+              "line=" + repr(line) + "; "
+              "p.write_text('\\n'.join(lines if line in lines else lines+[line])+'\\n'); "
+              "p.chmod(0o600)")
+    result = json.loads(qm('guest', 'exec', '223', '--timeout', '10', '--', '/usr/bin/python3', '-c', script))
+    if not result.get('exited') or result.get('exitcode') != 0:
+        raise RuntimeError('admin certificate authorization restore failed')
+
+
+def run():
+    # Permanent lock inode; never unlink it while other cron instances may wait.
+    with open('/run/decoy-reset.lock', 'a') as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            print('Reset already in progress')
+            return
+        validate()
+        changed = False
+        try:
+            command('systemctl', 'stop', 'decoy-ingress.service')
+            changed = True
+            command('systemctl', 'stop', 'decoy-watchdog.service')
+            guest('systemctl stop decoy.service decoy-network.service')
+            qm('stop', '223')
+            if 'status: stopped' not in qm('status', '223'):
+                raise RuntimeError('execution VM did not stop')
+            # Recheck after intake stops: never erase a concurrent capture-failure latch.
+            current, baseline = validate()
+            bounds(8 * 1024**3)
+            stamp = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+            ARCHIVES.mkdir(mode=0o700, exist_ok=True)
+            archive = ARCHIVES / (stamp + '-' + uuid.uuid4().hex)
+            archive.mkdir(mode=0o700)
+            sync_dir(ARCHIVES)
+            write_file(archive / 'vm223.conf', current.encode())
+            write_file(archive / 'baseline.conf', baseline.encode())
+            partial = archive / 'execution.qcow2.partial'
+            # Reserve file exclusively before qemu-img writes it; protected parent0700.
+            write_file(partial, b'')
+            with (archive / 'convert.log').open('xb') as output:
+                process = subprocess.Popen(['qemu-img', 'convert', '-c', '-f', 'raw', '-O',
+                                            'qcow2', str(DISK), str(partial)], stdout=output, stderr=output)
+                deadline = time.monotonic() + 900
+                try:
+                    while process.poll() is None:
+                        bounds()
+                        if time.monotonic() > deadline:
+                            raise RuntimeError('disk archive timeout')
+                        time.sleep(0.5)
+                    if process.returncode:
+                        raise RuntimeError('disk archive conversion failed; inspect retained convert.log')
+                finally:
+                    if process.poll() is None:
+                        process.kill()
+                        process.wait()
+                    output.flush()
+                    os.fsync(output.fileno())
+            bounds()
+            check = command('qemu-img', 'check', '--output=json', str(partial), timeout=300)
+            command('qemu-img', 'compare', '-f', 'raw', '-F', 'qcow2', str(DISK), str(partial), timeout=300)
+            write_file(archive / 'disk-check.json', check.encode())
+            disk_hash = digest(partial)
+            image = archive / 'execution.qcow2'
+            partial.rename(image)
+            sync_dir(archive)
+            pcap = ROOT / 'execution.pcap'
+            if pcap.is_symlink() or not pcap.is_file():
+                raise RuntimeError('pcap is missing or not a regular file')
+            pcap_hash = digest(pcap)
+            pcap_size = pcap.stat().st_size
+            if sum(p.stat().st_size for p in ARCHIVES.rglob('*') if p.is_file()) + pcap_size >= LIMIT:
+                raise RuntimeError('pcap would exceed archive budget')
+            pcap.rename(archive / 'execution.pcap')
+            sync_dir(ROOT)
+            sync_dir(archive)
+            metadata = {'timestamp_utc': stamp, 'vmid': 223, 'baseline': 'decoy-clean',
+                        'disk_sha256': disk_hash, 'pcap_sha256': pcap_hash,
+                        'disk_bytes': image.stat().st_size, 'pcap_bytes': pcap_size,
+                        'memory_preserved': False, 'state': 'validated_before_rollback'}
+            write_file(archive / 'manifest.json', (json.dumps(metadata, indent=2) + '\n').encode())
+            bounds()
+            # A preexisting STOP is never cleared. Any late latch aborts reset.
+            if (ROOT / 'STOP').exists():
+                raise RuntimeError('host STOP appeared during archive')
+            guest('test ! -e /var/lib/decoy/STOP')
+            qm('rollback', '223', 'decoy-clean', timeout=300)
+            valid_config(qm('config', '223'))
+            write_file(ROOT / 'execution.pcap', b'')
+            guest('test ! -e /var/lib/decoy/STOP && systemctl start decoy-network.service decoy.service && '
+                  'systemctl is-active --quiet decoy-network.service decoy.service')
+            qm('start', '223')
+            deadline = time.monotonic() + 90
+            while True:
+                try:
+                    ready = json.loads(qm('guest', 'exec', '223', '--timeout', '5', '--',
+                        '/bin/sh', '-c', 'test -f /etc/decoy-disposable-guest && systemctl is-active --quiet decoy-relay decoy-guest-network', timeout=10))
+                    if ready.get('exited') and ready.get('exitcode') == 0:
+                        break
+                except (subprocess.SubprocessError, ValueError):
+                    pass
+                if time.monotonic() >= deadline:
+                    raise RuntimeError('restored guest did not become ready')
+                time.sleep(2)
+            install_admin_ca()
+            command('systemctl', 'start', 'decoy-watchdog.service')
+            command('systemctl', 'start', 'decoy-ingress.service')
+            write_file(archive / 'reset-complete.json',
+                       json.dumps({'completed_time_ns': time.time_ns()}).encode())
+            print(f'Reset VM223; disk and packet evidence retained at {archive}')
+        except BaseException:
+            if changed:
+                # Best effort fail-closed; never resume intake or roll back on failure.
+                for args in [('systemctl', 'stop', 'decoy-ingress.service'),
+                             ('/usr/sbin/qm', 'stop', '223')]:
+                    try:
+                        command(*args)
+                    except Exception:
+                        pass
+                try:
+                    guest('systemctl stop decoy.service decoy-network.service')
+                except Exception:
+                    pass
+                try:
+                    write_file(ROOT / 'STOP', b'automatic reset failed; operator review required\n')
+                except FileExistsError:
+                    pass
+            raise
+        finally:
+            if changed:
+                command('systemctl', 'start', 'decoy-watchdog.service')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument('--run', action='store_true')
+    mode.add_argument('--check', action='store_true')
+    args = parser.parse_args()
+    os.umask(0o077)
+    if args.check:
+        validate()
+        bounds(8 * 1024**3)
+        print('Reset preflight passed; no changes made')
+    else:
+        run()