92 lines
3.2 KiB
Python
92 lines
3.2 KiB
Python
|
"""Local append-only event journal and inert, content-addressed evidence."""
|
||
|
|
import datetime
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import pathlib
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
|
||
|
|
|
||
|
|
class EvidenceError(OSError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
class Evidence:
|
||
|
|
def __init__(self, directory, quota=1024 ** 3):
|
||
|
|
self.directory = pathlib.Path(directory)
|
||
|
|
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
|
|
self.artifacts = self.directory / 'artifacts'
|
||
|
|
self.artifacts.mkdir(exist_ok=True, mode=0o700)
|
||
|
|
os.chmod(self.directory, 0o700)
|
||
|
|
os.chmod(self.artifacts, 0o700)
|
||
|
|
self.lock = threading.RLock()
|
||
|
|
self.quota = quota
|
||
|
|
self.used = sum(p.stat().st_size for p in self.directory.rglob('*') if p.is_file())
|
||
|
|
self.journal = os.open(self.directory / 'events.jsonl', os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
||
|
|
os.fchmod(self.journal, 0o600)
|
||
|
|
|
||
|
|
def reserve(self, size):
|
||
|
|
if self.used + size > self.quota:
|
||
|
|
raise OSError('Evidence quota reached; preserve/archive evidence before restarting')
|
||
|
|
|
||
|
|
def emit(self, event, **data):
|
||
|
|
now = time.time_ns()
|
||
|
|
entry = dict(schema_version=2, event=event, time_ns=now,
|
||
|
|
timestamp=datetime.datetime.fromtimestamp(now / 1e9, datetime.timezone.utc).isoformat(timespec='microseconds'),
|
||
|
|
**data)
|
||
|
|
raw = (json.dumps(entry, ensure_ascii=True) + '\n').encode()
|
||
|
|
with self.lock:
|
||
|
|
self.reserve(len(raw))
|
||
|
|
with os.fdopen(os.dup(self.journal), 'ab') as output:
|
||
|
|
output.write(raw)
|
||
|
|
output.flush()
|
||
|
|
os.fsync(output.fileno())
|
||
|
|
self.used += len(raw)
|
||
|
|
|
||
|
|
def save(self, data):
|
||
|
|
digest = hashlib.sha256(data).hexdigest()
|
||
|
|
path = self.artifacts / (digest + '.bin')
|
||
|
|
with self.lock:
|
||
|
|
if not path.exists():
|
||
|
|
self.reserve(len(data))
|
||
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||
|
|
try:
|
||
|
|
with os.fdopen(fd, 'wb') as output:
|
||
|
|
output.write(data)
|
||
|
|
output.flush()
|
||
|
|
os.fsync(output.fileno())
|
||
|
|
except BaseException:
|
||
|
|
path.unlink(missing_ok=True) # do not leave a partial blob with a full-content hash
|
||
|
|
raise
|
||
|
|
self.used += len(data)
|
||
|
|
return dict(sha256=digest, bytes=len(data), artifact='artifacts/' + path.name)
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
os.close(self.journal)
|
||
|
|
|
||
|
|
|
||
|
|
class RecordingReader:
|
||
|
|
"""Retain only bytes consumed by the HTTP parser, after TLS decryption."""
|
||
|
|
def __init__(self, source, limit):
|
||
|
|
self.source, self.limit = source, limit
|
||
|
|
self.data = bytearray()
|
||
|
|
self.truncated = False
|
||
|
|
|
||
|
|
def record(self, data):
|
||
|
|
room = max(0, self.limit - len(self.data))
|
||
|
|
self.data.extend(data[:room])
|
||
|
|
self.truncated |= len(data) > room
|
||
|
|
return data
|
||
|
|
|
||
|
|
def read(self, *args):
|
||
|
|
return self.record(self.source.read(*args))
|
||
|
|
|
||
|
|
def read1(self, *args):
|
||
|
|
return self.record(self.source.read1(*args))
|
||
|
|
|
||
|
|
def readline(self, *args):
|
||
|
|
return self.record(self.source.readline(*args))
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self.source.close()
|