261 lines
11 KiB
Python
261 lines
11 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Hourly VM223 reset, preserving its full disk and pcap first. Never prunes evidence.
|
||
|
|
|
||
|
|
--check is read-only. --run stops intake and VM223, archives disk/pcap, validates,
|
||
|
|
then rolls back decoy-clean. On any failure it leaves VM223/intake stopped and
|
||
|
|
sets the host STOP latch. RAM is not preserved; collector evidence is untouched.
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
import datetime
|
||
|
|
import fcntl
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
from pathlib import Path
|
||
|
|
import shutil
|
||
|
|
import stat
|
||
|
|
import subprocess
|
||
|
|
import time
|
||
|
|
import uuid
|
||
|
|
|
||
|
|
ROOT = Path('/var/lib/vz/honeypot')
|
||
|
|
ARCHIVES = ROOT / 'archives'
|
||
|
|
DISK = Path('/dev/pve/vm-223-disk-0')
|
||
|
|
LIMIT = 16 * 1024**3
|
||
|
|
RESERVE = 4 * 1024**3
|
||
|
|
|
||
|
|
|
||
|
|
def command(*args, timeout=60):
|
||
|
|
return subprocess.run(args, check=True, capture_output=True, text=True,
|
||
|
|
timeout=timeout).stdout
|
||
|
|
|
||
|
|
|
||
|
|
def qm(*args, timeout=60):
|
||
|
|
return command('/usr/sbin/qm', *args, timeout=timeout)
|
||
|
|
|
||
|
|
|
||
|
|
def guest(script):
|
||
|
|
result = json.loads(qm('guest', 'exec', '222', '--timeout', '20', '--',
|
||
|
|
'/bin/sh', '-c', script, timeout=25))
|
||
|
|
if not result.get('exited') or result.get('exitcode') != 0:
|
||
|
|
raise RuntimeError('collector guest operation failed')
|
||
|
|
return result.get('out-data', '')
|
||
|
|
|
||
|
|
|
||
|
|
def sync_dir(path):
|
||
|
|
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||
|
|
try:
|
||
|
|
os.fsync(fd)
|
||
|
|
finally:
|
||
|
|
os.close(fd)
|
||
|
|
|
||
|
|
|
||
|
|
def write_file(path, data):
|
||
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||
|
|
with os.fdopen(fd, 'wb') as stream:
|
||
|
|
stream.write(data)
|
||
|
|
stream.flush()
|
||
|
|
os.fsync(stream.fileno())
|
||
|
|
sync_dir(path.parent)
|
||
|
|
|
||
|
|
|
||
|
|
def bounds(min_free=RESERVE):
|
||
|
|
used = sum(p.stat().st_size for p in ARCHIVES.rglob('*') if p.is_file()) if ARCHIVES.exists() else 0
|
||
|
|
if used >= LIMIT:
|
||
|
|
raise RuntimeError('archive 16GiB budget reached; evidence retained')
|
||
|
|
if shutil.disk_usage(ROOT).free < min_free:
|
||
|
|
raise RuntimeError('host free-space reserve reached; evidence retained')
|
||
|
|
|
||
|
|
|
||
|
|
def valid_config(config):
|
||
|
|
values = dict(line.split(': ', 1) for line in config.splitlines() if ': ' in line)
|
||
|
|
if (values.get('name') != 'decoy-execution' or
|
||
|
|
not values.get('scsi0', '').startswith('local-lvm:vm-223-disk-0,') or
|
||
|
|
'restrict=on' not in values.get('args', '') or
|
||
|
|
'file=/var/lib/vz/honeypot/execution.pcap' not in values.get('args', '') or
|
||
|
|
any(k.startswith('net') and k[3:].isdigit() for k in values)):
|
||
|
|
raise RuntimeError('VM223 identity or isolation mismatch')
|
||
|
|
|
||
|
|
|
||
|
|
def validate():
|
||
|
|
if os.geteuid() != 0:
|
||
|
|
raise RuntimeError('run on designated Proxmox host as root')
|
||
|
|
if ROOT.is_symlink() or not ROOT.is_dir() or (ARCHIVES.exists() and ARCHIVES.is_symlink()):
|
||
|
|
raise RuntimeError('invalid evidence directory')
|
||
|
|
if (ROOT / 'STOP').exists():
|
||
|
|
raise RuntimeError('preexisting host STOP latch requires operator review')
|
||
|
|
if not stat.S_ISBLK(DISK.stat().st_mode):
|
||
|
|
raise RuntimeError('execution disk is not a block device')
|
||
|
|
current = qm('config', '223')
|
||
|
|
baseline = qm('config', '223', '--snapshot', 'decoy-clean')
|
||
|
|
valid_config(current)
|
||
|
|
valid_config(baseline)
|
||
|
|
guest('test -f /etc/decoy-collector && test ! -e /var/lib/decoy/STOP')
|
||
|
|
return current, baseline
|
||
|
|
|
||
|
|
|
||
|
|
def digest(path):
|
||
|
|
hashed = hashlib.sha256()
|
||
|
|
with path.open('rb') as stream:
|
||
|
|
while chunk := stream.read(1024**2):
|
||
|
|
bounds()
|
||
|
|
hashed.update(chunk)
|
||
|
|
os.fsync(stream.fileno())
|
||
|
|
return hashed.hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def install_admin_ca():
|
||
|
|
public = ROOT / 'admin-ca.authorized'
|
||
|
|
if not public.exists():
|
||
|
|
return
|
||
|
|
line = public.read_text().strip()
|
||
|
|
if not line.startswith('cert-authority,principals="decoy-admin",restrict,pty ssh-ed25519 ') or len(line) > 4096 or '\n' in line:
|
||
|
|
raise RuntimeError('invalid dedicated admin CA authorization')
|
||
|
|
script = ("import pathlib,os; p=pathlib.Path('/root/.ssh/authorized_keys'); "
|
||
|
|
"p.parent.mkdir(mode=0o700,exist_ok=True); "
|
||
|
|
"lines=p.read_text().splitlines() if p.exists() else []; "
|
||
|
|
"line=" + repr(line) + "; "
|
||
|
|
"p.write_text('\\n'.join(lines if line in lines else lines+[line])+'\\n'); "
|
||
|
|
"p.chmod(0o600)")
|
||
|
|
result = json.loads(qm('guest', 'exec', '223', '--timeout', '10', '--', '/usr/bin/python3', '-c', script))
|
||
|
|
if not result.get('exited') or result.get('exitcode') != 0:
|
||
|
|
raise RuntimeError('admin certificate authorization restore failed')
|
||
|
|
|
||
|
|
|
||
|
|
def run():
|
||
|
|
# Permanent lock inode; never unlink it while other cron instances may wait.
|
||
|
|
with open('/run/decoy-reset.lock', 'a') as lock:
|
||
|
|
try:
|
||
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
|
|
except BlockingIOError:
|
||
|
|
print('Reset already in progress')
|
||
|
|
return
|
||
|
|
validate()
|
||
|
|
changed = False
|
||
|
|
try:
|
||
|
|
command('systemctl', 'stop', 'decoy-ingress.service')
|
||
|
|
changed = True
|
||
|
|
command('systemctl', 'stop', 'decoy-watchdog.service')
|
||
|
|
guest('systemctl stop decoy.service decoy-network.service')
|
||
|
|
qm('stop', '223')
|
||
|
|
if 'status: stopped' not in qm('status', '223'):
|
||
|
|
raise RuntimeError('execution VM did not stop')
|
||
|
|
# Recheck after intake stops: never erase a concurrent capture-failure latch.
|
||
|
|
current, baseline = validate()
|
||
|
|
bounds(8 * 1024**3)
|
||
|
|
stamp = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||
|
|
ARCHIVES.mkdir(mode=0o700, exist_ok=True)
|
||
|
|
archive = ARCHIVES / (stamp + '-' + uuid.uuid4().hex)
|
||
|
|
archive.mkdir(mode=0o700)
|
||
|
|
sync_dir(ARCHIVES)
|
||
|
|
write_file(archive / 'vm223.conf', current.encode())
|
||
|
|
write_file(archive / 'baseline.conf', baseline.encode())
|
||
|
|
partial = archive / 'execution.qcow2.partial'
|
||
|
|
# Reserve file exclusively before qemu-img writes it; protected parent0700.
|
||
|
|
write_file(partial, b'')
|
||
|
|
with (archive / 'convert.log').open('xb') as output:
|
||
|
|
process = subprocess.Popen(['qemu-img', 'convert', '-c', '-f', 'raw', '-O',
|
||
|
|
'qcow2', str(DISK), str(partial)], stdout=output, stderr=output)
|
||
|
|
deadline = time.monotonic() + 900
|
||
|
|
try:
|
||
|
|
while process.poll() is None:
|
||
|
|
bounds()
|
||
|
|
if time.monotonic() > deadline:
|
||
|
|
raise RuntimeError('disk archive timeout')
|
||
|
|
time.sleep(0.5)
|
||
|
|
if process.returncode:
|
||
|
|
raise RuntimeError('disk archive conversion failed; inspect retained convert.log')
|
||
|
|
finally:
|
||
|
|
if process.poll() is None:
|
||
|
|
process.kill()
|
||
|
|
process.wait()
|
||
|
|
output.flush()
|
||
|
|
os.fsync(output.fileno())
|
||
|
|
bounds()
|
||
|
|
check = command('qemu-img', 'check', '--output=json', str(partial), timeout=300)
|
||
|
|
command('qemu-img', 'compare', '-f', 'raw', '-F', 'qcow2', str(DISK), str(partial), timeout=300)
|
||
|
|
write_file(archive / 'disk-check.json', check.encode())
|
||
|
|
disk_hash = digest(partial)
|
||
|
|
image = archive / 'execution.qcow2'
|
||
|
|
partial.rename(image)
|
||
|
|
sync_dir(archive)
|
||
|
|
pcap = ROOT / 'execution.pcap'
|
||
|
|
if pcap.is_symlink() or not pcap.is_file():
|
||
|
|
raise RuntimeError('pcap is missing or not a regular file')
|
||
|
|
pcap_hash = digest(pcap)
|
||
|
|
pcap_size = pcap.stat().st_size
|
||
|
|
if sum(p.stat().st_size for p in ARCHIVES.rglob('*') if p.is_file()) + pcap_size >= LIMIT:
|
||
|
|
raise RuntimeError('pcap would exceed archive budget')
|
||
|
|
pcap.rename(archive / 'execution.pcap')
|
||
|
|
sync_dir(ROOT)
|
||
|
|
sync_dir(archive)
|
||
|
|
metadata = {'timestamp_utc': stamp, 'vmid': 223, 'baseline': 'decoy-clean',
|
||
|
|
'disk_sha256': disk_hash, 'pcap_sha256': pcap_hash,
|
||
|
|
'disk_bytes': image.stat().st_size, 'pcap_bytes': pcap_size,
|
||
|
|
'memory_preserved': False, 'state': 'validated_before_rollback'}
|
||
|
|
write_file(archive / 'manifest.json', (json.dumps(metadata, indent=2) + '\n').encode())
|
||
|
|
bounds()
|
||
|
|
# A preexisting STOP is never cleared. Any late latch aborts reset.
|
||
|
|
if (ROOT / 'STOP').exists():
|
||
|
|
raise RuntimeError('host STOP appeared during archive')
|
||
|
|
guest('test ! -e /var/lib/decoy/STOP')
|
||
|
|
qm('rollback', '223', 'decoy-clean', timeout=300)
|
||
|
|
valid_config(qm('config', '223'))
|
||
|
|
write_file(ROOT / 'execution.pcap', b'')
|
||
|
|
guest('test ! -e /var/lib/decoy/STOP && systemctl start decoy-network.service decoy.service && '
|
||
|
|
'systemctl is-active --quiet decoy-network.service decoy.service')
|
||
|
|
qm('start', '223')
|
||
|
|
deadline = time.monotonic() + 90
|
||
|
|
while True:
|
||
|
|
try:
|
||
|
|
ready = json.loads(qm('guest', 'exec', '223', '--timeout', '5', '--',
|
||
|
|
'/bin/sh', '-c', 'test -f /etc/decoy-disposable-guest && systemctl is-active --quiet decoy-relay decoy-guest-network', timeout=10))
|
||
|
|
if ready.get('exited') and ready.get('exitcode') == 0:
|
||
|
|
break
|
||
|
|
except (subprocess.SubprocessError, ValueError):
|
||
|
|
pass
|
||
|
|
if time.monotonic() >= deadline:
|
||
|
|
raise RuntimeError('restored guest did not become ready')
|
||
|
|
time.sleep(2)
|
||
|
|
install_admin_ca()
|
||
|
|
command('systemctl', 'start', 'decoy-watchdog.service')
|
||
|
|
command('systemctl', 'start', 'decoy-ingress.service')
|
||
|
|
write_file(archive / 'reset-complete.json',
|
||
|
|
json.dumps({'completed_time_ns': time.time_ns()}).encode())
|
||
|
|
print(f'Reset VM223; disk and packet evidence retained at {archive}')
|
||
|
|
except BaseException:
|
||
|
|
if changed:
|
||
|
|
# Best effort fail-closed; never resume intake or roll back on failure.
|
||
|
|
for args in [('systemctl', 'stop', 'decoy-ingress.service'),
|
||
|
|
('/usr/sbin/qm', 'stop', '223')]:
|
||
|
|
try:
|
||
|
|
command(*args)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
try:
|
||
|
|
guest('systemctl stop decoy.service decoy-network.service')
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
try:
|
||
|
|
write_file(ROOT / 'STOP', b'automatic reset failed; operator review required\n')
|
||
|
|
except FileExistsError:
|
||
|
|
pass
|
||
|
|
raise
|
||
|
|
finally:
|
||
|
|
if changed:
|
||
|
|
command('systemctl', 'start', 'decoy-watchdog.service')
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||
|
|
mode.add_argument('--run', action='store_true')
|
||
|
|
mode.add_argument('--check', action='store_true')
|
||
|
|
args = parser.parse_args()
|
||
|
|
os.umask(0o077)
|
||
|
|
if args.check:
|
||
|
|
validate()
|
||
|
|
bounds(8 * 1024**3)
|
||
|
|
print('Reset preflight passed; no changes made')
|
||
|
|
else:
|
||
|
|
run()
|