Retain mixed evaluation indexing costs and partial failure evidence

tools/mixed-eval/evaluate.py

@@ -9,6 +9,8 @@ from pathlib import Path
 import shutil
 import subprocess
 import tempfile
+import time
+import platform
 
 ROOT = Path(__file__).resolve().parents[2]
 FIXTURE = ROOT / 'fixtures/mixed'
@@ -91,7 +93,33 @@ def run(args):
     spec = importlib.util.spec_from_file_location('searchi_benchmark', ROOT / 'tools/embeddinggemma2/benchmark.py')
     benchmark = importlib.util.module_from_spec(spec); spec.loader.exec_module(benchmark)
     args.output.parent.mkdir(parents=True, exist_ok=True)
-    rows = []
+    rows, stages = [], []
+    provenance = {'fixture_sha256': lock['fixture_sha256'], 'model_manifest_sha256': digest(manifest_path),
+                  'model': manifest, 'device': 'cpu', 'cpu_threads': 1,
+                  'cli_sha256': digest(args.cli), 'worker_sha256': digest(args.worker),
+                  'platform': platform.platform(), 'python': platform.python_version()}
+    def checkpoint():
+        args.output.write_text(json.dumps({**provenance, 'complete': False, 'stages': stages, 'rows': rows}, indent=2, ensure_ascii=False), encoding='utf-8')
+    def stage(name, command, timeout):
+        started = time.perf_counter()
+        try:
+            result = subprocess.run(command, check=False, timeout=timeout, capture_output=True, text=True, encoding='utf-8')
+        except subprocess.TimeoutExpired as error:
+            def text(value):
+                return value.decode('utf-8', errors='replace') if isinstance(value, bytes) else value or ''
+            stages.append({'name': name, 'elapsed_ms': (time.perf_counter() - started) * 1000,
+                           'timed_out': True, 'stdout': text(error.stdout), 'stderr': text(error.stderr)})
+            checkpoint()
+            raise
+        entry = {'name': name, 'elapsed_ms': (time.perf_counter() - started) * 1000,
+                 'returncode': result.returncode, 'stdout': result.stdout, 'stderr': result.stderr}
+        try:
+            entry['report'] = json.loads(result.stdout)
+        except json.JSONDecodeError:
+            pass
+        stages.append(entry); checkpoint()
+        result.check_returncode()
+    checkpoint()
     with tempfile.TemporaryDirectory(prefix='searchi-mixed-') as temporary:
         base = Path(temporary); corpus = base / 'corpus'; corpus.mkdir(); database = base / 'index.sqlite'
         names = {}
@@ -99,10 +127,10 @@ def run(args):
             source = (FIXTURE / asset['path']).resolve()
             shutil.copyfile(source, corpus / source.name)
             names[source.name] = asset['id']
-        subprocess.run([str(args.cli), 'index', '--db', str(database), '--root', str(corpus)], check=True, timeout=300)
-        subprocess.run([str(args.cli), 'embed', '--db', str(database), '--model-dir', str(args.models),
+        stage('lexical_index', [str(args.cli), 'index', '--db', str(database), '--root', str(corpus)], 300)
+        stage('semantic_index', [str(args.cli), 'embed', '--db', str(database), '--model-dir', str(args.models),
                         '--embedding-worker', str(args.worker), '--cpu-threads', '1',
-                        '--ffmpeg', str(args.ffmpeg), '--ffprobe', str(args.ffprobe)], check=True, timeout=3600)
+                        '--ffmpeg', str(args.ffmpeg), '--ffprobe', str(args.ffprobe)], 3600)
         for mode in ('lexical_only', 'hybrid'):
             command = [str(args.cli), 'mcp', '--db', str(database)]
             if mode == 'hybrid':
@@ -129,12 +157,12 @@ def run(args):
                         if content['mode'] != mode:
                             raise RuntimeError('Unexpected semantic fallback; not valid quality evidence')
                         hits = [{'asset': names[hit['filename']], 'score': hit['score'], 'match_kind': hit['match_kind'], 'source': hit.get('source')} for hit in content['results']]
-                        rows.append({'mode': mode, 'scope': scope, 'query': query, 'hits': hits, 'score': score(query, hits), 'roundtrip_ms': elapsed})
+                        rows.append({'mode': mode, 'scope': scope, 'query': query, 'hits': hits, 'score': score(query, hits), 'roundtrip_ms': elapsed, 'metrics': content.get('metrics')})
                         # Retain partial actual evidence if a later request fails.
-                        args.output.write_text(json.dumps({'complete': False, 'rows': rows}, indent=2, ensure_ascii=False), encoding='utf-8')
+                        checkpoint()
             finally:
                 session.close()
-    report = {'complete': True, 'recorded_at': datetime.now(timezone.utc).isoformat(), 'fixture_sha256': lock['fixture_sha256'],
+    report = {**provenance, 'stages': stages, 'complete': True, 'recorded_at': datetime.now(timezone.utc).isoformat(), 'fixture_sha256': lock['fixture_sha256'],
               'model_manifest_sha256': digest(manifest_path), 'model': manifest,
               'device': 'cpu', 'cpu_threads': 1, 'cli_sha256': digest(args.cli), 'worker_sha256': digest(args.worker),
               'scope_note': 'Small mixed diagnostic, reused previously tested media labels; file-level relevance only. No temporal-localization or production-scale claims. MCP roundtrip excludes initialization and is not UI latency.',