From 7b99db9e2524085cbfbf398ee088d51c650ca21c Mon Sep 17 00:00:00 2001 From: CitrixPot Date: Mon, 28 Sep 2026 20:13:25 +0200 Subject: [PATCH] viewer: raw-bytes prefilter to skip JSON parsing on non-matching lines --- viewer.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/viewer.py b/viewer.py index 076bd0c..4e5aa94 100644 --- a/viewer.py +++ b/viewer.py @@ -219,17 +219,25 @@ class Handler(http.server.BaseHTTPRequestHandler): need = (page + 1) * size rows, scanned = [], 0 path = self.server.journal + # Cheap raw-bytes prefilter: skip JSON parsing for lines that cannot match (event/q/peer/payload). + event_token = ('"event": "%s"' % kind).encode() if kind else None + payload_token = b'"evidence"' if payload_only else None if path.is_file(): - # ponytail: reverse scan is bounded by page depth, not file size, for unfiltered paging; - # add a sqlite FTS index only if filtered deep paging on multi-GB journals gets slow. + # ponytail: still O(file) for a rare term on a multi-GB journal; add a sqlite FTS5 index + # (stdlib, incremental) only once the journal reaches a few hundred MB. for raw in reverse_lines(path): scanned += 1 - line = raw.decode('utf-8', 'replace') - low = line.lower() - if any(term not in low for term in terms): + low = raw.lower() if (terms or peer) else None + if terms and any(term.encode() not in low for term in terms): + continue + if peer and peer.encode() not in low: + continue + if event_token and event_token not in raw: + continue + if payload_token and payload_token not in raw: continue try: - row = json.loads(line) + row = json.loads(raw) except ValueError: continue if kind and row.get('event') != kind: