feat: add multimodal search and granular performance diagnostics

tools/media-eval/.gitignore

@@ -0,0 +1,4 @@
+.venv/
+target/
+_*.py
+__pycache__/

tools/media-eval/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "searchi-media-eval"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[workspace]
+
+[dependencies]
+anyhow = "1"
+serde_json = "1"
+sha2 = "0.10"
+searchi-semantic = { path = "../../crates/searchi-semantic" }
+searchi-core = { path = "../../crates/searchi-core" }
+searchi-service = { path = "../../crates/searchi-service" }
+searchi-media = { path = "../../crates/searchi-media" }
+tempfile = "3"

tools/media-eval/README.md

@@ -0,0 +1,35 @@
+# Real-media quality evaluation
+
+This independent tool owns no production/UI code. It feeds real pixels and decoded PCM to the actual Google CPU worker, keeps frozen source labels outside inference input, and reports ranking/localization failures.
+
+```powershell
+$env:SEARCHI_FORCE_CPU = '1'
+python tools/media-eval/download.py
+python tools/media-eval/prepare.py --ffmpeg .searchi/media-runtime/bin/ffmpeg.exe --ffprobe .searchi/media-runtime/bin/ffprobe.exe
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin searchi-media-eval -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe fixtures/media/reference-results.json
+python tools/media-eval/score.py fixtures/media/reference-results.json fixtures/media/reference-summary.json
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata -- fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/metadata-results.json
+python tools/media-eval/score.py fixtures/media/metadata-results.json fixtures/media/metadata-summary.json
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin pipeline -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/pipeline-30s-results.json 30
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin pipeline -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/pipeline-10s-results.json 10
+python tools/media-eval/score.py fixtures/media/pipeline-30s-results.json fixtures/media/pipeline-30s-summary.json
+python tools/media-eval/score.py fixtures/media/pipeline-10s-results.json fixtures/media/pipeline-10s-summary.json
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin image_budgets -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe fixtures/media/image-budget-results.json
+python tools/media-eval/score_budgets.py fixtures/media/image-budget-results.json fixtures/media/image-budget-summary.json
+python -m unittest discover -s tools/media-eval -p test_*.py
+```
+
+The standalone Cargo workspace keeps evaluation dependencies separate. Native inference and FFmpeg decoding use CPU only and one thread. No test enables a GPU provider. The model/worker/decoder paths above refer to explicitly provisioned local developer artifacts; they are not end-user setup requirements.
+
+The dense reference samples video every two seconds and PCM in thirty-second windows. Photos use 140 image tokens and frames 70. The separate comparison fixes the same 12 photos/24 queries and varies only photo token budgets **70, 280, 1120**. It does not change labels or choose a threshold after observing scores.
+
+## Interpretation
+
+- Recall@1/@5 and MRR rank distinct source files, choosing each file's best segment. Per-category/per-language metrics remain separate.
+- Segment overlap credits a returned segment that actually intersects the labelled event in the correct file. Point hits require the returned start timestamp itself to fall inside the event. A thirty-second speech window may pass overlap while opening far too early; both metrics are required.
+- No-answer queries remain separate from positive recall. The tool exposes returned neighbors and scores without inventing an abstention threshold. A reject threshold needs independent calibration and held-out validation.
+- Direct model candidate filters are **reference pool constraints**, not evidence that production filters work. Artist/title identity cannot be recovered reliably from a performance embedding; exact metadata rows are explicitly unsupported until evaluated through Searchi's real lexical metadata lane.
+- `metadata` tests the actual FFprobe/guarded core FTS lane; `pipeline` tests the actual unified Searchi media generation and hybrid ranking. The optional final argument varies audio window seconds (30 versus 10) while retaining two-second overlap, the same real files, decoder and frozen labels. It does not change production defaults.
+- Model agreement with another implementation, synthetic document-seed scores, and this small-media benchmark each answer different questions. None is production quality certification.
+
+Detailed provenance is in [the fixture README](../../fixtures/media/README.md). The final assessment belongs in [MEDIA-QUALITY.md](../../docs/MEDIA-QUALITY.md).

tools/media-eval/download.py

@@ -0,0 +1,32 @@
+"""Reacquire the locked licensed corpus. Updated bytes are never accepted."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import urllib.request
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, default=Path("fixtures/media"))
+    args = parser.parse_args()
+    root = args.root.resolve()
+    fixture_path = root / "fixture.json"
+    lock = json.loads((root / "labels.lock.json").read_text())
+    assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == lock["fixture_sha256"]
+    for asset in json.loads(fixture_path.read_text(encoding="utf-8"))["assets"]:
+        destination = (root / asset["path"]).resolve()
+        assert destination.is_relative_to(root)
+        if destination.is_file() and hashlib.sha256(destination.read_bytes()).hexdigest() == asset["sha256"]:
+            continue
+        request = urllib.request.Request(asset["source_url"], headers={"User-Agent": "SearchiMediaEvaluation/1.0"})
+        with urllib.request.urlopen(request, timeout=60) as response:
+            data = response.read(asset["bytes"] + 1)
+        assert len(data) == asset["bytes"] and hashlib.sha256(data).hexdigest() == asset["sha256"], f"Publisher changed {asset['id']}; keep frozen corpus, investigate a new version"
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(data)
+        print("Verified", asset["id"])
+
+
+if __name__ == "__main__":
+    main()

tools/media-eval/prepare.py

@@ -0,0 +1,62 @@
+"""Prepare real CPU-decoded media candidates after verifying frozen labels.
+
+No generated captions, placeholder vectors, or model calls occur in preparation.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+
+
+def verify(root):
+    fixture_path = root / "fixture.json"
+    lock = json.loads((root / "labels.lock.json").read_text())
+    assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == lock["fixture_sha256"], "Frozen labels changed"
+    fixture = json.loads(fixture_path.read_text(encoding="utf-8"))
+    for asset in fixture["assets"]:
+        path = (root / asset["path"]).resolve()
+        assert path.is_relative_to(root.resolve()), "Asset escaped corpus root"
+        assert path.stat().st_size == asset["bytes"], asset["id"]
+        assert hashlib.sha256(path.read_bytes()).hexdigest() == asset["sha256"], asset["id"]
+    return fixture
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, default=Path("fixtures/media"))
+    parser.add_argument("--ffmpeg", type=Path, required=True)
+    parser.add_argument("--ffprobe", type=Path, required=True)
+    args = parser.parse_args()
+    root = args.root.resolve()
+    fixture = verify(root)
+    derived = root / "derived"
+    derived.mkdir(exist_ok=True)
+    candidates = []
+    for asset in fixture["assets"]:
+        source = root / asset["path"]
+        if asset["kind"] == "image":
+            candidates.append(dict(asset=asset["id"], path=str(source), modality="image", token_budget=140))
+            continue
+        probe = json.loads(subprocess.check_output([str(args.ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(source)], text=True))
+        duration = float(probe["format"]["duration"])
+        if asset["kind"] == "video":
+            for stamp in range(0, int(duration) + 1, 2):
+                output = derived / f"{asset['id']}-frame-{stamp:04}.png"
+                subprocess.run([str(args.ffmpeg), "-hide_banner", "-loglevel", "error", "-hwaccel", "none", "-threads", "1", "-ss", str(stamp), "-i", str(source), "-frames:v", "1", "-vf", "scale=480:-1", "-threads", "1", "-y", str(output)], check=True)
+                if output.is_file():
+                    candidates.append(dict(asset=asset["id"], path=str(output), modality="image", token_budget=70, start_s=stamp, end_s=min(stamp + 2, duration)))
+        if any(stream["codec_type"] == "audio" for stream in probe["streams"]):
+            for stamp in range(0, int(duration) + 1, 30):
+                output = derived / f"{asset['id']}-audio-{stamp:04}.f32"
+                subprocess.run([str(args.ffmpeg), "-hide_banner", "-loglevel", "error", "-hwaccel", "none", "-threads", "1", "-ss", str(stamp), "-i", str(source), "-t", "30", "-vn", "-ac", "1", "-ar", "16000", "-f", "f32le", "-threads", "1", "-y", str(output)], check=True)
+                if output.stat().st_size:
+                    assert output.stat().st_size <= 480000 * 4, "Audio exceeds worker budget"
+                    candidates.append(dict(asset=asset["id"], path=str(output), modality="audio", start_s=stamp, end_s=min(stamp + 30, duration)))
+    manifest = dict(fixture_sha256=hashlib.sha256((root / "fixture.json").read_bytes()).hexdigest(), ffmpeg_version=subprocess.check_output([str(args.ffmpeg), "-version"], text=True).splitlines()[0], policy="CPU-only dense reference: photos140/video70 tokens, uniform2s frames, mono16kHz30s PCM", candidates=candidates)
+    (derived / "candidates.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+    print(json.dumps(dict(candidates=len(candidates), images=sum(item["modality"] == "image" for item in candidates), audio=sum(item["modality"] == "audio" for item in candidates))))
+
+
+if __name__ == "__main__":
+    main()

tools/media-eval/score.py

@@ -0,0 +1,114 @@
+"""Score frozen relevance without fitting an abstention threshold."""
+import argparse
+from collections import defaultdict
+import hashlib
+import json
+from pathlib import Path
+
+
+def temporal_hit(candidate, query, point=False):
+    asset = candidate.get("asset")
+    if asset not in query.get("expected", []):
+        return False
+    if query.get("required_modality") and candidate.get("modality") != query["required_modality"]:
+        return False
+    ranges = query.get("intervals", {}).get(asset, [])
+    start = candidate.get("start_s")
+    end = candidate.get("end_s", start)
+    if start is None:
+        return False
+    if point:
+        return any(left <= start <= right for left, right in ranges)
+    return any(min(end, right) > max(start, left) for left, right in ranges)
+
+
+def timestamp_distance(candidate, query):
+    """Raw seconds from returned start to the nearest labelled event interval."""
+    if candidate.get("asset") not in query.get("expected", []) or candidate.get("start_s") is None:
+        return None
+    if query.get("required_modality") and candidate.get("modality") != query["required_modality"]:
+        return None
+    ranges = query.get("intervals", {}).get(candidate["asset"], [])
+    stamp = candidate["start_s"]
+    return min((max(left - stamp, stamp - right, 0) for left, right in ranges), default=None)
+
+
+def metrics(rows):
+    positive = [row for row in rows if row["query"]["expected"] and not row.get("unsupported")]
+    negative = [row for row in rows if not row["query"]["expected"] and not row.get("unsupported")]
+    temporal = [row for row in positive if row["query"]["category"].startswith("video-")]
+    distances = sorted(distance for row in temporal if row["file_rankings"] and (distance := timestamp_distance(row["file_rankings"][0]["candidate"], row["query"])) is not None)
+    ranks = []
+    for row in positive:
+        expected = row["query"]["expected"]
+        ranks.append(next((i + 1 for i, hit in enumerate(row["file_rankings"]) if hit["candidate"]["asset"] in expected), None))
+    fraction = lambda count, total: count / total if total else None
+    return dict(
+        positive_queries=len(positive),
+        unsupported_queries=sum(bool(row.get("unsupported")) for row in rows),
+        recall_at1=fraction(sum(rank == 1 for rank in ranks), len(ranks)),
+        recall_at5=fraction(sum(rank is not None and rank <= 5 for rank in ranks), len(ranks)),
+        mrr=fraction(sum(1 / rank if rank else 0 for rank in ranks), len(ranks)),
+        temporal_queries=len(temporal),
+        segment_overlap_hit_at1=fraction(sum(bool(row["file_rankings"]) and temporal_hit(row["file_rankings"][0]["candidate"], row["query"]) for row in temporal), len(temporal)),
+        timestamp_point_hit_at1=fraction(sum(bool(row["file_rankings"]) and temporal_hit(row["file_rankings"][0]["candidate"], row["query"], point=True) for row in temporal), len(temporal)),
+        segment_overlap_hit_at5=fraction(sum(any(temporal_hit(hit["candidate"], row["query"]) for hit in row["segment_top5"]) for row in temporal), len(temporal)),
+        timestamp_distance_median_s=distances[len(distances) // 2] if distances else None,
+        timestamp_distance_max_s=max(distances) if distances else None,
+        no_answer_queries=len(negative),
+        no_answer_return_rate=fraction(sum(bool(row["file_rankings"]) for row in negative), len(negative)),
+    )
+
+
+def summarize(report):
+    rows = report["rows"]
+    by_category = defaultdict(list)
+    by_language = defaultdict(list)
+    failures = []
+    for row in rows:
+        query = row["query"]
+        by_category[query["category"]].append(row)
+        by_language[query["language"]].append(row)
+        if row.get("unsupported"):
+            failures.append(dict(query=query["id"], category="unsupported-query-capability", detail=row["unsupported"]))
+            continue
+        hits = row["file_rankings"]
+        if not query["expected"]:
+            if hits:
+                failures.append(dict(query=query["id"], category="no-answer-nearest-neighbor-return", winner=hits[0]["candidate"]["asset"], score=hits[0]["score"]))
+        elif not hits or hits[0]["candidate"]["asset"] not in query["expected"]:
+            winner = hits[0]["candidate"]["asset"] if hits else None
+            failures.append(dict(query=query["id"], category="hard-negative-confusion" if winner in query.get("hard_negatives", []) else "retrieval-top1-miss", winner=winner, expected=query["expected"]))
+        elif query["category"].startswith("video-") and not temporal_hit(hits[0]["candidate"], query, point=True):
+            candidate = hits[0]["candidate"]
+            wrong_channel = query.get("required_modality") and candidate.get("modality") != query["required_modality"]
+            failures.append(dict(query=query["id"], category="wrong-media-channel" if wrong_channel else "timestamp-localization-miss", candidate=candidate, expected_intervals=query["intervals"], timestamp_distance_s=timestamp_distance(candidate, query)))
+    durations = sorted(row["query_ms"] for row in rows if "query_ms" in row)
+    return dict(
+        fixture_sha256=report["fixture_sha256"], worker=report["worker"],
+        overall=metrics(rows), by_category={key: metrics(value) for key, value in sorted(by_category.items())},
+        by_language={key: metrics(value) for key, value in sorted(by_language.items())},
+        encoding_failures=report["encoding_failures"], encoding_ms=report["encoding_ms"],
+        query_p50_ms=durations[len(durations) // 2] if durations else None,
+        query_p95_ms=durations[min(len(durations) * 95 // 100, len(durations) - 1)] if durations else None,
+        failures=failures,
+        source_scope=report.get("scope", "Direct dense model baseline"),
+        interpretation="Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality.",
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("report", type=Path)
+    parser.add_argument("output", type=Path)
+    args = parser.parse_args()
+    report = json.loads(args.report.read_text(encoding="utf-8"))
+    fixture = Path("fixtures/media/fixture.json")
+    assert report["fixture_sha256"] == hashlib.sha256(fixture.read_bytes()).hexdigest()
+    summary = summarize(report)
+    args.output.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+    print(json.dumps(summary["overall"]))
+
+
+if __name__ == "__main__":
+    main()

tools/media-eval/score_budgets.py

@@ -0,0 +1,38 @@
+"""Summarize predeclared image-budget experiments without fitting thresholds."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+from score import metrics
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("report", type=Path)
+    parser.add_argument("output", type=Path)
+    args = parser.parse_args()
+    report = json.loads(args.report.read_text(encoding="utf-8"))
+    fixture_hash = hashlib.sha256(Path("fixtures/media/fixture.json").read_bytes()).hexdigest()
+    if report["fixture_sha256"] != fixture_hash:
+        raise ValueError("Frozen fixture changed")
+    if [budget["token_budget"] for budget in report["budgets"]] != [70, 280, 1120]:
+        raise ValueError("Expected predeclared budgets in order")
+    summary = {
+        "fixture_sha256": fixture_hash,
+        "worker": report["worker"],
+        "photos": report["photos"],
+        "queries": report["queries"],
+        "query_embedding_ms": report["query_embedding_ms"],
+        "budgets": [dict(token_budget=budget["token_budget"],
+                         encoding_ms=budget["encoding_ms"],
+                         metrics=metrics(budget["rows"]))
+                    for budget in report["budgets"]],
+        "interpretation": "Same frozen 12 photos and 24 correlated EN/DE queries. Query embeddings shared across budgets. Encoding time is CPU batch cost; ranking-only time excludes inference. Equal small-corpus recall does not prove equal quality on larger or more difficult images.",
+    }
+    args.output.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+    print(json.dumps(summary["budgets"]))
+
+
+if __name__ == "__main__":
+    main()

tools/media-eval/src/bin/image_budgets.rs

@@ -0,0 +1,100 @@
+//! Same frozen real photos/queries across predeclared token budgets.
+use anyhow::{ensure, Result};
+use searchi_semantic::{Device, GemmaEmbedder, GEMMA_MEDIA_FINGERPRINT};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 4,
+        "Supply CORPUS_ROOT MODEL_DIR CPU_WORKER OUTPUT_JSON"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU only"
+    );
+    let root = PathBuf::from(&args[0]);
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(lock["fixture_sha256"] == hash, "Frozen labels changed");
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    let photos: Vec<_> = fixture["assets"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|asset| asset["kind"] == "image")
+        .collect();
+    let queries: Vec<_> = fixture["queries"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|query| query["category"] == "image-memory")
+        .collect();
+    for photo in &photos {
+        let content = fs::read(root.join(photo["path"].as_str().unwrap()))?;
+        ensure!(
+            photo["sha256"] == format!("{:x}", Sha256::digest(content)),
+            "Frozen photo changed"
+        );
+    }
+    let mut model = GemmaEmbedder::load_with_device(&args[1], &args[2], 1, Device::Cpu)?;
+    let mut query_vectors = Vec::new();
+    let query_start = Instant::now();
+    for query in &queries {
+        query_vectors.push(model.embed_query(query["text"].as_str().unwrap())?);
+    }
+    let query_embedding_ms = query_start.elapsed().as_secs_f64() * 1000.0;
+    let mut budgets = Vec::new();
+    for budget in [70, 280, 1120] {
+        let start = Instant::now();
+        let mut encoded = Vec::new();
+        for photo in &photos {
+            let path = root.join(photo["path"].as_str().unwrap());
+            let image = model
+                .embed_images_with_metadata(&[&path], budget)?
+                .remove(0);
+            encoded.push(image);
+        }
+        let encoding_ms = start.elapsed().as_secs_f64() * 1000.0;
+        let mut rows = Vec::new();
+        for (query, query_vector) in queries.iter().zip(&query_vectors) {
+            let rank_start = Instant::now();
+            let mut ranked: Vec<_> = photos
+                .iter()
+                .zip(&encoded)
+                .map(|(photo, image)| {
+                    let score: f64 = query_vector
+                        .iter()
+                        .zip(&image.vector)
+                        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+                        .sum();
+                    json!({"candidate":{"asset":photo["id"],"modality":"image"},"score":score})
+                })
+                .collect();
+            ranked.sort_by(|a, b| {
+                b["score"]
+                    .as_f64()
+                    .unwrap()
+                    .total_cmp(&a["score"].as_f64().unwrap())
+            });
+            rows.push(json!({"query":query,"file_rankings":ranked,"segment_top5":[],"ranking_only_ms":rank_start.elapsed().as_secs_f64()*1000.0}));
+        }
+        let resolutions: Vec<_> = photos.iter().zip(&encoded).map(|(photo,image)| json!({"asset":photo["id"],"encoded_width":image.encoded_width,"encoded_height":image.encoded_height,"token_budget":image.token_budget})).collect();
+        budgets.push(json!({"token_budget":budget,"encoding_ms":encoding_ms,"images":resolutions,"rows":rows}));
+        println!(
+            "Budget {budget}: {} actual photo encodings, {} frozen queries",
+            photos.len(),
+            queries.len()
+        );
+    }
+    fs::write(
+        &args[3],
+        serde_json::to_vec_pretty(
+            &json!({"fixture_sha256":hash,"worker":model.info,"media_fingerprint":GEMMA_MEDIA_FINGERPRINT,"photos":photos.len(),"queries":queries.len(),"query_embedding_ms":query_embedding_ms,"budgets":budgets,"scope":"12-photo-only token-budget comparison; same 24 independently frozen queries, query embeddings computed once and shared across budgets; ranking_only_ms excludes inference; no retrieval threshold or label tuning. Does not certify production search."}),
+        )?,
+    )?;
+    Ok(())
+}

tools/media-eval/src/bin/metadata.rs

@@ -0,0 +1,78 @@
+//! Fast actual metadata/keyword check, separate from embedding semantics.
+use anyhow::{ensure, Result};
+use searchi_core::SearchIndex;
+use searchi_media::{probe, FfmpegTools, MediaIdentity, MediaLimits};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(args.len() == 4, "Supply CORPUS FFMPEG FFPROBE OUTPUT");
+    let root = PathBuf::from(&args[0]).canonicalize()?;
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(lock["fixture_sha256"] == hash, "Frozen labels changed");
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    let tools = FfmpegTools::new(PathBuf::from(&args[1]), PathBuf::from(&args[2]))?;
+    let limits = MediaLimits::default();
+    let mut index = SearchIndex::open(":memory:")?;
+    index.index_folder(root.join("assets"))?;
+    let extraction_start = Instant::now();
+    for asset in fixture["assets"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|asset| asset["kind"] != "image")
+    {
+        let identity = MediaIdentity::read(
+            &root.join(asset["path"].as_str().unwrap()),
+            limits.max_file_bytes,
+        )?;
+        ensure!(
+            asset["sha256"] == identity.sha256,
+            "Frozen source asset changed"
+        );
+        let source = probe(&tools, &identity.path, &limits)?;
+        let tags = source
+            .tags
+            .iter()
+            .map(|(key, value)| format!("{key}: {value}"))
+            .collect::<Vec<_>>()
+            .join("\n");
+        index.set_media_metadata(
+            &identity.path.to_string_lossy(),
+            &tags,
+            source.duration_ms,
+            identity.bytes,
+            identity.modified_ns,
+        )?;
+    }
+    let extraction_ms = extraction_start.elapsed().as_secs_f64() * 1000.0;
+    let mut rows = Vec::new();
+    for query in fixture["queries"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|query| query.get("metadata").is_some())
+    {
+        let start = Instant::now();
+        let hits = index.search(query["text"].as_str().unwrap(), 20)?;
+        let query_ms = start.elapsed().as_secs_f64() * 1000.0;
+        let file_rankings: Vec<_> = hits.iter().map(|hit| {
+            let asset = fixture["assets"].as_array().unwrap().iter().find(|asset| PathBuf::from(asset["path"].as_str().unwrap()).file_name().unwrap().to_string_lossy()==hit.filename).unwrap();
+            json!({"candidate":{"asset":asset["id"],"modality":"metadata","match_kind":hit.match_kind},"score":hit.score})
+        }).collect();
+        rows.push(
+            json!({"query":query,"file_rankings":file_rankings,"segment_top5":[],"query_ms":query_ms}),
+        );
+    }
+    fs::write(
+        &args[3],
+        serde_json::to_vec_pretty(
+            &json!({"fixture_sha256":hash,"worker":null,"encoding_ms":extraction_ms,"encoding_failures":[],"rows":rows,"scope":"Actual FFprobe stream/format metadata + guarded Searchi core FTS; lexical identity queries, no embedding fallback; encoding_ms measures metadata extraction, not model inference"}),
+        )?,
+    )?;
+    Ok(())
+}

tools/media-eval/src/bin/pipeline.rs

@@ -0,0 +1,83 @@
+//! End-to-end corpus indexing and querying through actual Searchi APIs.
+use anyhow::{ensure, Result};
+use searchi_core::{SearchIndex, SearchOptions};
+use searchi_media::{FfmpegTools, MediaLimits};
+use searchi_service::HybridSearch;
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 6 || args.len() == 7,
+        "Supply CORPUS MODEL CPU_WORKER FFMPEG FFPROBE OUTPUT_JSON [AUDIO_WINDOW_SECONDS]"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU only"
+    );
+    let root = PathBuf::from(&args[0]).canonicalize()?;
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(lock["fixture_sha256"] == hash, "Frozen labels changed");
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    for asset in fixture["assets"].as_array().unwrap() {
+        let content = fs::read(root.join(asset["path"].as_str().unwrap()))?;
+        ensure!(
+            asset["sha256"] == format!("{:x}", Sha256::digest(content)),
+            "Asset changed"
+        );
+    }
+    let temporary = tempfile::tempdir()?;
+    let db = temporary.path().join("index.sqlite");
+    let mut lexical = SearchIndex::open(&db)?;
+    let filename_index = lexical.index_folder(root.join("assets"))?;
+    let mut hybrid = HybridSearch::open(&db, &args[1], &args[2], 1)?;
+    let tools = FfmpegTools::new(PathBuf::from(&args[3]), PathBuf::from(&args[4]))?;
+    let start = Instant::now();
+    let mut limits = MediaLimits::default();
+    if let Some(seconds) = args.get(6) {
+        let seconds: u32 = seconds.to_string_lossy().parse()?;
+        ensure!(
+            (5..=30).contains(&seconds),
+            "Audio window must be 5–30 seconds"
+        );
+        limits.audio_window_ms = seconds * 1000;
+    }
+    let index_report = hybrid.index_embeddings_with_media(&tools, &limits)?;
+    let encoding_ms = start.elapsed().as_secs_f64() * 1000.0;
+    println!("Actual pipeline indexed {} chunks", index_report.chunks);
+    hybrid.warmup()?;
+    let mut rows = Vec::new();
+    for query in fixture["queries"].as_array().unwrap() {
+        if query["filters"].get("time_range_s").is_some() {
+            rows.push(json!({"query":query,"unsupported":"Production query API has no temporal-range filter yet"}));
+            continue;
+        }
+        let mut options = SearchOptions::default();
+        if let Some(extension) = query["filters"]["extension"].as_str() {
+            options.extension = Some(extension.into());
+        }
+        let start = Instant::now();
+        let hits = hybrid.search(query["text"].as_str().unwrap(), 20, &options)?;
+        let query_ms = start.elapsed().as_secs_f64() * 1000.0;
+        let file_rankings: Vec<_> = hits.iter().map(|hit| {
+            let asset = fixture["assets"].as_array().unwrap().iter().find(|asset| PathBuf::from(asset["path"].as_str().unwrap()).file_name().unwrap().to_string_lossy() == hit.filename).unwrap();
+            let modality = if hit.match_kind.contains("audio") { "audio" } else if hit.match_kind == "media_metadata" { "metadata" } else { "image" };
+            let mut candidate = json!({"asset":asset["id"],"modality":modality,"match_kind":hit.match_kind,"source":hit.source});
+            if let Some(source) = &hit.source { if source.kind == "timestamp_ms" {
+                let stamp = f64::from(source.number)/1000.0;
+                candidate["start_s"] = json!(stamp);
+                candidate["end_s"] = json!((stamp + if modality == "audio" {f64::from(limits.audio_window_ms)/1000.0} else {2.0}).min(asset["duration_s"].as_f64().unwrap_or(stamp)));
+                candidate["interval_note"] = json!("End inferred from configured pipeline window policy, actual API exposes only start timestamp");
+            }}
+            json!({"candidate":candidate,"score":hit.score})
+        }).collect();
+        rows.push(json!({"query":query,"segment_top5":file_rankings.iter().take(5).cloned().collect::<Vec<_>>(),"file_rankings":file_rankings,"query_ms":query_ms}));
+    }
+    let report = json!({"fixture_sha256":hash,"worker":hybrid.worker_info(),"index_report":index_report,"filename_index":filename_index,"encoding_ms":encoding_ms,"audio_window_ms":limits.audio_window_ms,"audio_overlap_ms":limits.audio_overlap_ms,"encoding_failures":index_report.failures,"rows":rows,"scope":"Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API"});
+    fs::write(&args[5], serde_json::to_vec_pretty(&report)?)?;
+    Ok(())
+}

tools/media-eval/src/main.rs

@@ -0,0 +1,153 @@
+//! Actual CPU Google embeddings only; labels never enter document/image vectors.
+use anyhow::{ensure, Context, Result};
+use searchi_semantic::{Device, GemmaEmbedder, GEMMA_FINGERPRINT, GEMMA_MEDIA_FINGERPRINT};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 4,
+        "Supply CORPUS_ROOT MODEL_DIR CPU_WORKER OUTPUT_JSON"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU-only benchmark requires SEARCHI_FORCE_CPU=1"
+    );
+    let root = PathBuf::from(&args[0]);
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(
+        lock["fixture_sha256"] == hash,
+        "Independent labels changed after freezing"
+    );
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    let candidates: Value =
+        serde_json::from_slice(&fs::read(root.join("derived/candidates.json"))?)?;
+    ensure!(
+        candidates["fixture_sha256"] == hash,
+        "Prepared media belongs to different labels"
+    );
+    let mut model = GemmaEmbedder::load_with_device(&args[1], &args[2], 1, Device::Cpu)?;
+    ensure!(
+        model.info.device.to_lowercase().contains("cpu"),
+        "Worker did not report CPU"
+    );
+    let start = Instant::now();
+    let mut vectors = Vec::new();
+    let mut encoded = Vec::new();
+    let mut encoding_failures = Vec::new();
+    for candidate in candidates["candidates"]
+        .as_array()
+        .context("candidates absent")?
+    {
+        let path = PathBuf::from(candidate["path"].as_str().context("path absent")?);
+        let result = if candidate["modality"] == "image" {
+            model
+                .embed_images_with_budget(
+                    &[&path],
+                    candidate["token_budget"].as_u64().unwrap() as u32,
+                )
+                .map(|mut rows| rows.remove(0))
+        } else {
+            let bytes = fs::read(&path)?;
+            ensure!(bytes.len() % 4 == 0, "PCM alignment invalid");
+            let samples: Vec<f32> = bytes
+                .as_chunks::<4>()
+                .0
+                .iter()
+                .map(|chunk| f32::from_le_bytes(*chunk))
+                .collect();
+            model.embed_audio_pcm(&samples)
+        };
+        match result {
+            Ok(vector) => {
+                vectors.push(vector);
+                encoded.push(candidate.clone());
+            }
+            Err(error) => {
+                encoding_failures.push(json!({"candidate": candidate, "error": error.to_string()}))
+            }
+        }
+        println!(
+            "Encoded {}/{} candidates",
+            encoded.len(),
+            candidates["candidates"].as_array().unwrap().len()
+        );
+    }
+    let index_ms = start.elapsed().as_secs_f64() * 1000.0;
+    let mut rows = Vec::new();
+    for query in fixture["queries"].as_array().context("queries absent")? {
+        if query.get("metadata").is_some() {
+            rows.push(json!({"query":query,"unsupported":"Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"}));
+            continue;
+        }
+        let start = Instant::now();
+        let embedding = model.embed_query(query["text"].as_str().context("query text absent")?)?;
+        let mut ranked = Vec::new();
+        for (candidate, vector) in encoded.iter().zip(&vectors) {
+            if let Some(modality) = query.get("required_modality") {
+                if candidate["modality"] != *modality {
+                    continue;
+                }
+            }
+            if let Some(filters) = query.get("filters") {
+                if let Some(asset) = filters.get("asset") {
+                    if candidate["asset"] != *asset {
+                        continue;
+                    }
+                }
+                if let Some(extension) = filters.get("extension").and_then(Value::as_str) {
+                    let asset = fixture["assets"]
+                        .as_array()
+                        .unwrap()
+                        .iter()
+                        .find(|asset| asset["id"] == candidate["asset"])
+                        .unwrap();
+                    if !asset["path"]
+                        .as_str()
+                        .unwrap()
+                        .ends_with(&format!(".{extension}"))
+                    {
+                        continue;
+                    }
+                }
+                if let Some(range) = filters.get("time_range_s") {
+                    let stamp = candidate["start_s"].as_f64().unwrap_or(-1.0);
+                    if stamp < range[0].as_f64().unwrap() || stamp > range[1].as_f64().unwrap() {
+                        continue;
+                    }
+                }
+            }
+            ensure!(
+                embedding.len() == vector.len(),
+                "Mixed embedding dimensions"
+            );
+            let score: f64 = embedding
+                .iter()
+                .zip(vector)
+                .map(|(a, b)| f64::from(*a) * f64::from(*b))
+                .sum();
+            ranked.push(json!({"candidate":candidate,"score":score}));
+        }
+        ranked.sort_by(|a, b| {
+            b["score"]
+                .as_f64()
+                .unwrap()
+                .total_cmp(&a["score"].as_f64().unwrap())
+        });
+        let mut seen = std::collections::HashSet::new();
+        let file_rankings: Vec<_> = ranked
+            .iter()
+            .filter(|row| seen.insert(row["candidate"]["asset"].as_str().unwrap()))
+            .take(20)
+            .cloned()
+            .collect();
+        rows.push(json!({"query":query,"file_rankings":file_rankings,"segment_top5":ranked.into_iter().take(5).collect::<Vec<_>>(),"query_ms":start.elapsed().as_secs_f64()*1000.0}));
+    }
+    let report = json!({"fixture_sha256":hash,"worker":model.info,"text_fingerprint":GEMMA_FINGERPRINT,"media_fingerprint":GEMMA_MEDIA_FINGERPRINT,"policy":candidates["policy"],"ffmpeg_version":candidates["ffmpeg_version"],"encoding_ms":index_ms,"encoded_candidates":encoded.len(),"encoding_failures":encoding_failures,"rows":rows,"scope":"Direct real-model dense reference baseline; candidate filters here are benchmark reference constraints, not production-filter certification; no rejection threshold is tuned"});
+    fs::write(&args[3], serde_json::to_vec_pretty(&report)?)?;
+    Ok(())
+}

tools/media-eval/test_score.py

@@ -0,0 +1,39 @@
+import unittest
+from score import metrics, temporal_hit, timestamp_distance
+
+
+class ScoringTests(unittest.TestCase):
+    def test_wrong_file_cannot_receive_temporal_credit(self):
+        query = dict(expected=["clip01"], intervals={"clip01": [[3, 5]]})
+        self.assertFalse(temporal_hit(dict(asset="clip02", start_s=3, end_s=5), query))
+
+    def test_coarse_audio_overlap_does_not_count_as_precise_timestamp(self):
+        query = dict(expected=["speech"], intervals={"speech": [[13, 16]]})
+        candidate = dict(asset="speech", start_s=0, end_s=30)
+        self.assertTrue(temporal_hit(candidate, query))
+        self.assertFalse(temporal_hit(candidate, query, point=True))
+        self.assertFalse(temporal_hit(dict(asset="speech", start_s=16, end_s=30), query))
+        self.assertEqual(timestamp_distance(candidate, query), 13)
+        self.assertEqual(timestamp_distance(dict(asset="speech", start_s=15), query), 0)
+
+    def test_visual_frame_cannot_receive_credit_for_a_spoken_topic(self):
+        query = dict(expected=["movie"], intervals={"movie": [[10, 15]]}, required_modality="audio")
+        candidate = dict(asset="movie", modality="image", start_s=12, end_s=14)
+        self.assertFalse(temporal_hit(candidate, query))
+        self.assertIsNone(timestamp_distance(candidate, query))
+
+    def test_no_answers_and_unsupported_cases_do_not_inflate_recall(self):
+        positive = dict(query=dict(expected=["good"], category="image-memory"), file_rankings=[dict(candidate=dict(asset="bad")), dict(candidate=dict(asset="good"))])
+        negative = dict(query=dict(expected=[], category="content-no-answer"), file_rankings=[dict(candidate=dict(asset="bad"))])
+        unsupported = dict(query=dict(expected=["good"], category="exact-metadata"), unsupported="metadata unavailable")
+        result = metrics([positive, negative, unsupported])
+        self.assertEqual(result["positive_queries"], 1)
+        self.assertEqual(result["recall_at1"], 0)
+        self.assertEqual(result["recall_at5"], 1)
+        self.assertEqual(result["mrr"], .5)
+        self.assertEqual(result["no_answer_return_rate"], 1)
+        self.assertEqual(result["unsupported_queries"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main()