diff --git a/tools/mixed-eval/README.md b/tools/mixed-eval/README.md index e95e940..7f7f7fc 100644 --- a/tools/mixed-eval/README.md +++ b/tools/mixed-eval/README.md @@ -20,7 +20,10 @@ differ. Inference runs must be coordinated with other CPU measurements. Expect media indexing to take minutes; its explicit one-hour bound is a safety ceiling, not a performance target. Indexing and query errors remain visible. Temporary corpora/indexes are cleaned after the owned sessions close. Partial query results -are retained in the requested output if a later request fails. +are retained in the requested output if a later request fails. Indexing stage +stdout/stderr, JSON reports, failures, timeouts and elapsed time are also retained. +`complete: true` means evaluation finished, never that release quality passed. +The query rows retain native basic metrics separately from MCP roundtrip time. Reported metrics are file-level top1, reciprocal rank and recall@5 after file identity deduplication. Hard negatives in the first five and no-answer queries diff --git a/tools/mixed-eval/evaluate.py b/tools/mixed-eval/evaluate.py index 2337547..acdf3ba 100644 --- a/tools/mixed-eval/evaluate.py +++ b/tools/mixed-eval/evaluate.py @@ -9,6 +9,8 @@ from pathlib import Path import shutil import subprocess import tempfile +import time +import platform ROOT = Path(__file__).resolve().parents[2] FIXTURE = ROOT / 'fixtures/mixed' @@ -91,7 +93,33 @@ def run(args): spec = importlib.util.spec_from_file_location('searchi_benchmark', ROOT / 'tools/embeddinggemma2/benchmark.py') benchmark = importlib.util.module_from_spec(spec); spec.loader.exec_module(benchmark) args.output.parent.mkdir(parents=True, exist_ok=True) - rows = [] + rows, stages = [], [] + provenance = {'fixture_sha256': lock['fixture_sha256'], 'model_manifest_sha256': digest(manifest_path), + 'model': manifest, 'device': 'cpu', 'cpu_threads': 1, + 'cli_sha256': digest(args.cli), 'worker_sha256': digest(args.worker), + 'platform': platform.platform(), 'python': platform.python_version()} + def checkpoint(): + args.output.write_text(json.dumps({**provenance, 'complete': False, 'stages': stages, 'rows': rows}, indent=2, ensure_ascii=False), encoding='utf-8') + def stage(name, command, timeout): + started = time.perf_counter() + try: + result = subprocess.run(command, check=False, timeout=timeout, capture_output=True, text=True, encoding='utf-8') + except subprocess.TimeoutExpired as error: + def text(value): + return value.decode('utf-8', errors='replace') if isinstance(value, bytes) else value or '' + stages.append({'name': name, 'elapsed_ms': (time.perf_counter() - started) * 1000, + 'timed_out': True, 'stdout': text(error.stdout), 'stderr': text(error.stderr)}) + checkpoint() + raise + entry = {'name': name, 'elapsed_ms': (time.perf_counter() - started) * 1000, + 'returncode': result.returncode, 'stdout': result.stdout, 'stderr': result.stderr} + try: + entry['report'] = json.loads(result.stdout) + except json.JSONDecodeError: + pass + stages.append(entry); checkpoint() + result.check_returncode() + checkpoint() with tempfile.TemporaryDirectory(prefix='searchi-mixed-') as temporary: base = Path(temporary); corpus = base / 'corpus'; corpus.mkdir(); database = base / 'index.sqlite' names = {} @@ -99,10 +127,10 @@ def run(args): source = (FIXTURE / asset['path']).resolve() shutil.copyfile(source, corpus / source.name) names[source.name] = asset['id'] - subprocess.run([str(args.cli), 'index', '--db', str(database), '--root', str(corpus)], check=True, timeout=300) - subprocess.run([str(args.cli), 'embed', '--db', str(database), '--model-dir', str(args.models), + stage('lexical_index', [str(args.cli), 'index', '--db', str(database), '--root', str(corpus)], 300) + stage('semantic_index', [str(args.cli), 'embed', '--db', str(database), '--model-dir', str(args.models), '--embedding-worker', str(args.worker), '--cpu-threads', '1', - '--ffmpeg', str(args.ffmpeg), '--ffprobe', str(args.ffprobe)], check=True, timeout=3600) + '--ffmpeg', str(args.ffmpeg), '--ffprobe', str(args.ffprobe)], 3600) for mode in ('lexical_only', 'hybrid'): command = [str(args.cli), 'mcp', '--db', str(database)] if mode == 'hybrid': @@ -129,12 +157,12 @@ def run(args): if content['mode'] != mode: raise RuntimeError('Unexpected semantic fallback; not valid quality evidence') hits = [{'asset': names[hit['filename']], 'score': hit['score'], 'match_kind': hit['match_kind'], 'source': hit.get('source')} for hit in content['results']] - rows.append({'mode': mode, 'scope': scope, 'query': query, 'hits': hits, 'score': score(query, hits), 'roundtrip_ms': elapsed}) + rows.append({'mode': mode, 'scope': scope, 'query': query, 'hits': hits, 'score': score(query, hits), 'roundtrip_ms': elapsed, 'metrics': content.get('metrics')}) # Retain partial actual evidence if a later request fails. - args.output.write_text(json.dumps({'complete': False, 'rows': rows}, indent=2, ensure_ascii=False), encoding='utf-8') + checkpoint() finally: session.close() - report = {'complete': True, 'recorded_at': datetime.now(timezone.utc).isoformat(), 'fixture_sha256': lock['fixture_sha256'], + report = {**provenance, 'stages': stages, 'complete': True, 'recorded_at': datetime.now(timezone.utc).isoformat(), 'fixture_sha256': lock['fixture_sha256'], 'model_manifest_sha256': digest(manifest_path), 'model': manifest, 'device': 'cpu', 'cpu_threads': 1, 'cli_sha256': digest(args.cli), 'worker_sha256': digest(args.worker), 'scope_note': 'Small mixed diagnostic, reused previously tested media labels; file-level relevance only. No temporal-localization or production-scale claims. MCP roundtrip excludes initialization and is not UI latency.',