viewer: raw-bytes prefilter to skip JSON parsing on non-matching lines
This commit is contained in:
1 parent
d74bd7ce56
commit
7b99db9e25
1 file changed
+14
-6
@@ -219,17 +219,25 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
|||||||
need = (page + 1) * size
|
need = (page + 1) * size
|
||||||
rows, scanned = [], 0
|
rows, scanned = [], 0
|
||||||
path = self.server.journal
|
path = self.server.journal
|
||||||
|
# Cheap raw-bytes prefilter: skip JSON parsing for lines that cannot match (event/q/peer/payload).
|
||||||
|
event_token = ('"event": "%s"' % kind).encode() if kind else None
|
||||||
|
payload_token = b'"evidence"' if payload_only else None
|
||||||
if path.is_file():
|
if path.is_file():
|
||||||
# ponytail: reverse scan is bounded by page depth, not file size, for unfiltered paging;
|
# ponytail: still O(file) for a rare term on a multi-GB journal; add a sqlite FTS5 index
|
||||||
# add a sqlite FTS index only if filtered deep paging on multi-GB journals gets slow.
|
# (stdlib, incremental) only once the journal reaches a few hundred MB.
|
||||||
for raw in reverse_lines(path):
|
for raw in reverse_lines(path):
|
||||||
scanned += 1
|
scanned += 1
|
||||||
line = raw.decode('utf-8', 'replace')
|
low = raw.lower() if (terms or peer) else None
|
||||||
low = line.lower()
|
if terms and any(term.encode() not in low for term in terms):
|
||||||
if any(term not in low for term in terms):
|
continue
|
||||||
|
if peer and peer.encode() not in low:
|
||||||
|
continue
|
||||||
|
if event_token and event_token not in raw:
|
||||||
|
continue
|
||||||
|
if payload_token and payload_token not in raw:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
row = json.loads(line)
|
row = json.loads(raw)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
if kind and row.get('event') != kind:
|
if kind and row.get('event') != kind:
|
||||||
|
|||||||
Reference in new issue
Block a user