143 lines
6.9 KiB
Python
143 lines
6.9 KiB
Python
|
"""Storage inventory and saved message export analysis."""
|
||
|
|
import argparse
|
||
|
|
import collections
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
from .common import BASE, PROFILE, normalized
|
||
|
|
|
||
|
|
def messages(path):
|
||
|
|
with path.open(encoding='utf-8-sig') as stream:
|
||
|
|
if path.suffix.lower() == '.jsonl':
|
||
|
|
for number, line in enumerate(stream, 1):
|
||
|
|
if not line.strip():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
row = json.loads(line)
|
||
|
|
except json.JSONDecodeError as ex:
|
||
|
|
raise ValueError(f'Invalid JSON at line {number}: {ex}') from ex
|
||
|
|
if not isinstance(row, dict):
|
||
|
|
raise ValueError(f'Line {number} is not a message object')
|
||
|
|
yield row
|
||
|
|
else:
|
||
|
|
data = json.load(stream)
|
||
|
|
if isinstance(data, list):
|
||
|
|
rows = data
|
||
|
|
elif isinstance(data, dict):
|
||
|
|
rows = next((data[key] for key in ('messages', 'models', 'results')
|
||
|
|
if isinstance(data.get(key), list)), None)
|
||
|
|
if rows is None:
|
||
|
|
raise ValueError('Expected messages, models or results array')
|
||
|
|
else:
|
||
|
|
raise ValueError('Expected an array or a message report')
|
||
|
|
for row in rows:
|
||
|
|
if not isinstance(row, dict):
|
||
|
|
raise ValueError('Message array contains a non-object value')
|
||
|
|
yield row
|
||
|
|
|
||
|
|
def content(row):
|
||
|
|
return '\n'.join(row[key] for key in ('indexed_text', 'body', 'caption', 'text')
|
||
|
|
if isinstance(row.get(key), str))
|
||
|
|
|
||
|
|
def inventory(profile):
|
||
|
|
entries = []
|
||
|
|
for pattern in ('IndexedDB/*whatsapp*', 'Local Storage/leveldb', 'Session Storage'):
|
||
|
|
for folder in profile.glob(pattern):
|
||
|
|
files = []
|
||
|
|
for file in folder.iterdir() if folder.is_dir() else [folder]:
|
||
|
|
try:
|
||
|
|
if file.is_file():
|
||
|
|
files.append({'name': file.name, 'bytes': file.stat().st_size})
|
||
|
|
except OSError:
|
||
|
|
continue
|
||
|
|
entries.append({'path': str(folder), 'files': files})
|
||
|
|
return {'profile': str(profile), 'exists': profile.exists(), 'storage_inventory': entries,
|
||
|
|
'coverage': 'File names and sizes only; no decrypted messages.'}
|
||
|
|
|
||
|
|
def main(argv=None):
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
sub = parser.add_subparsers(dest='command', required=True)
|
||
|
|
inv = sub.add_parser('inventory', help='List local Desktop storage files and sizes')
|
||
|
|
inv.add_argument('--profile', type=Path, default=PROFILE)
|
||
|
|
for name in ('stats', 'search'):
|
||
|
|
cmd = sub.add_parser(name, help='Analyze an existing JSON/JSONL message export')
|
||
|
|
cmd.add_argument('input', type=Path)
|
||
|
|
cmd.add_argument('--output', type=Path)
|
||
|
|
if name == 'search':
|
||
|
|
cmd.add_argument('query')
|
||
|
|
cmd.add_argument('--chat-id')
|
||
|
|
cmd.add_argument('--limit', type=int, default=100)
|
||
|
|
cmd.add_argument('--redact-text', action='store_true')
|
||
|
|
sub.add_parser('webview-test', help='Harmless JavaScript test in a separate local host')
|
||
|
|
live = sub.add_parser('whatsapp-test', help='Run eight fixed arithmetic checks in the running Desktop instance')
|
||
|
|
live.add_argument('--pid', type=int, help='Optional override; running Desktop host is detected automatically')
|
||
|
|
live.add_argument('--output', type=Path, default=BASE / 'analysis/webview2/live-arithmetic-result.json')
|
||
|
|
search = sub.add_parser('whatsapp-search', help='Use the running app native message search through WebView2')
|
||
|
|
search.add_argument('query')
|
||
|
|
search.add_argument('--pid', type=int, help='Optional override; running Desktop host is detected automatically')
|
||
|
|
search.add_argument('--limit', type=int, default=50)
|
||
|
|
search.add_argument('--output', type=Path, default=BASE / 'analysis/webview2/live-message-search.json')
|
||
|
|
args = parser.parse_args(argv)
|
||
|
|
if args.command == 'whatsapp-test':
|
||
|
|
from .live import main as run_live
|
||
|
|
return run_live(['--suite', 'math'] + (['--pid', str(args.pid)] if args.pid else []) + ['--output', str(args.output)])
|
||
|
|
if args.command == 'whatsapp-search':
|
||
|
|
if not args.query.strip() or len(args.query)>200 or not 1<=args.limit<=50:
|
||
|
|
parser.error('Search requires 1..200 characters and --limit 1..50')
|
||
|
|
from .native_search import run_live_probe
|
||
|
|
return run_live_probe(args.pid, args.output, args.query, args.limit)
|
||
|
|
if args.command == 'webview-test':
|
||
|
|
from .webview import run_webview_test
|
||
|
|
return run_webview_test()
|
||
|
|
if args.command == 'inventory':
|
||
|
|
result = inventory(args.profile)
|
||
|
|
elif args.command == 'stats':
|
||
|
|
count = with_text = 0
|
||
|
|
kinds = collections.Counter()
|
||
|
|
chats = set()
|
||
|
|
for row in messages(args.input):
|
||
|
|
count += 1
|
||
|
|
with_text += bool(content(row))
|
||
|
|
kinds[str(row.get('type', 'unknown'))] += 1
|
||
|
|
chat = row.get('chatId', row.get('chat_id'))
|
||
|
|
if chat is not None:
|
||
|
|
chats.add(str(chat))
|
||
|
|
result = {'source': str(args.input), 'messages': count, 'messages_with_text': with_text,
|
||
|
|
'chats': len(chats), 'types': dict(kinds), 'scope': 'Saved export only'}
|
||
|
|
else:
|
||
|
|
if not args.query.strip():
|
||
|
|
parser.error('query must not be empty')
|
||
|
|
if args.limit < 1:
|
||
|
|
parser.error('--limit must be positive')
|
||
|
|
found = []
|
||
|
|
total = 0
|
||
|
|
for row in messages(args.input):
|
||
|
|
chat = row.get('chatId', row.get('chat_id'))
|
||
|
|
if args.chat_id is not None and str(chat) != args.chat_id:
|
||
|
|
continue
|
||
|
|
text = content(row)
|
||
|
|
if normalized(args.query) not in normalized(text):
|
||
|
|
continue
|
||
|
|
total += 1
|
||
|
|
if len(found) >= args.limit:
|
||
|
|
continue
|
||
|
|
item = dict(row)
|
||
|
|
if args.redact_text:
|
||
|
|
# Allowlist metadata so nested fields cannot leak message content.
|
||
|
|
item = {key: value for key, value in row.items()
|
||
|
|
if key in ('id', 'chatId', 'chat_id', 'timestamp', 'type', 'nativeRowID', 'native_rowid')
|
||
|
|
and isinstance(value, (str, int, float, bool, type(None)))}
|
||
|
|
item['text_length'] = len(text)
|
||
|
|
found.append(item)
|
||
|
|
result = {'source': str(args.input), 'matches': total, 'returned': len(found),
|
||
|
|
'results': found, 'scope': 'Literal search of saved export only'}
|
||
|
|
serialized = json.dumps(result, ensure_ascii=False, indent=2)
|
||
|
|
output = getattr(args, 'output', None)
|
||
|
|
if output:
|
||
|
|
if output.resolve() == args.input.resolve():
|
||
|
|
parser.error('output must differ from input')
|
||
|
|
output.write_text(serialized + '\n', encoding='utf-8')
|
||
|
|
print(json.dumps({'output': str(output), 'command': args.command}))
|
||
|
|
else:
|
||
|
|
print(serialized)
|
||
|
|
return 0
|
||
|
|
|