viewer: raw-bytes prefilter to skip JSON parsing on non-matching lines
This commit is contained in:
1 parent
d74bd7ce56
commit
7b99db9e25
1 file changed
+14
-6
@@ -219,17 +219,25 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
need = (page + 1) * size
|
||||
rows, scanned = [], 0
|
||||
path = self.server.journal
|
||||
# Cheap raw-bytes prefilter: skip JSON parsing for lines that cannot match (event/q/peer/payload).
|
||||
event_token = ('"event": "%s"' % kind).encode() if kind else None
|
||||
payload_token = b'"evidence"' if payload_only else None
|
||||
if path.is_file():
|
||||
# ponytail: reverse scan is bounded by page depth, not file size, for unfiltered paging;
|
||||
# add a sqlite FTS index only if filtered deep paging on multi-GB journals gets slow.
|
||||
# ponytail: still O(file) for a rare term on a multi-GB journal; add a sqlite FTS5 index
|
||||
# (stdlib, incremental) only once the journal reaches a few hundred MB.
|
||||
for raw in reverse_lines(path):
|
||||
scanned += 1
|
||||
line = raw.decode('utf-8', 'replace')
|
||||
low = line.lower()
|
||||
if any(term not in low for term in terms):
|
||||
low = raw.lower() if (terms or peer) else None
|
||||
if terms and any(term.encode() not in low for term in terms):
|
||||
continue
|
||||
if peer and peer.encode() not in low:
|
||||
continue
|
||||
if event_token and event_token not in raw:
|
||||
continue
|
||||
if payload_token and payload_token not in raw:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
row = json.loads(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if kind and row.get('event') != kind:
|
||||
|
||||
Reference in new issue
Block a user