Add frozen shared-service speech retrieval and negative-control evaluator

tools/speech-eval/native-validation/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "searchi-speech-validation"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[workspace]
+
+[dependencies]
+anyhow = "1"
+serde_json = "1"
+sha2 = "0.10"
+rusqlite = { version = "0.32", features = ["bundled"] }
+searchi-core = { path = "../../../crates/searchi-core" }
+searchi-media = { path = "../../../crates/searchi-media" }
+searchi-service = { path = "../../../crates/searchi-service" }
+searchi-speech = { path = "../../../crates/searchi-speech" }
+
+[dev-dependencies]
+tempfile = "3"

tools/speech-eval/native-validation/README.md

@@ -0,0 +1,53 @@
+# Actual shared-service speech validation
+
+This internal evaluator uses the real `SpeechWorker`, shared service publication
+and actual Google EmbeddingGemma 2. It consumes the unchanged frozen
+`fixtures/speech-validation-v1` sources and queries. No fixture reference text,
+keywords or expected results enter recognition or indexed passages.
+
+Run from the repository root after committing the evaluator/source snapshot and
+obtaining the coordinator's exclusive CPU inference slot. The normal native
+runtime is forced to CPU here; the shown `cpu-v3` speech runtime is an internal
+test artifact and must be verified by its owner before execution.
+
+```powershell
+cargo test --manifest-path tools/speech-eval/native-validation/Cargo.toml --locked
+python -m unittest discover -s tools/speech-eval -p test_score_native_validation.py
+$env:SEARCHI_FORCE_CPU = '1'
+cargo run --manifest-path tools/speech-eval/native-validation/Cargo.toml --locked -- fixtures/speech-validation-v1/fixture.json .searchi/speech-validation-v1-seams/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/speech-runtime/cpu-v3/runtime/searchi-speech-worker.exe .searchi/speech/models/ggml-base.bin .searchi/speech/models/ggml-silero-v6.2.0.bin .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/speech-service-validation-v1
+python tools/speech-eval/score_native_validation.py .searchi/qa/speech-service-validation-v1/report.json fixtures/speech-validation-v1/fixture.json .searchi/qa/speech-service-validation-v1/scored.json
+```
+
+The output directory must not already exist. Individual native-call and query
+files, preflight, indexed state and case reports are immutable, with each file
+flushed before continuing. Failure writes an explicit incomplete artifact while
+preserving completed calls. Local databases/owned copied sources remain there
+for inspection. Model/runtime/source hashes, policy fingerprints and executing
+binary hash are recorded before inference. Freeze and rebuild shared sources
+before a scored run; on-disk source hashes alone cannot prove dependency compile
+provenance while another agent is editing.
+
+The core case retains all four prepared files and all ten frozen queries.
+Speech calls are matched against independently decoded source-window PCM hashes.
+Accepted native `SpeechTranscription` values or returned errors are stored exactly
+through the public client API. **Native wire bytes and stderr are not exposed by
+that API**, so these artifacts are not claimed to capture raw transport output.
+Published speech text/source hashes/times and byte-verified service excerpts are
+retained, and foreground search uses the actual query role with measured traces.
+The second case indexes both separate artificial seam sources using the same
+service window policy. It checks source coverage and publication bounds; it does
+not compare against continuously initialized full-source VAD.
+
+The scorer independently verifies complete native PCM-window coverage, every
+frozen query, result-source identity and negative controls. It reports positive
+file recall/MRR, absent queries returning candidates, recognized negative-control
+passages and literal publisher-concept substring coverage separately. Concept
+matching is a diagnostic, not WER or an independent timed transcript.
+
+There are no publisher phrase timestamps, so **no strict word/sentence point
+quality score is produced**. A timestamp merely inside the complete article is
+not precise localization success. Any excluded/failed source invalidates accepted
+comparison scoring, and hallucinations on music/silence remain measured failures.
+No provider, ranking weight, rejection threshold or production policy is chosen
+from this small experiment. The two readings share translated subject matter and
+the seam controls repeat real samples artificially.

tools/speech-eval/native-validation/src/main.rs

@@ -0,0 +1,378 @@
+//! Frozen actual shared-service speech validation; no reference text enters inference.
+use anyhow::{ensure, Context, Result};
+use searchi_core::SearchIndex;
+use searchi_media::{decode_audio_aligned, probe, AudioWindow, FfmpegTools, MediaLimits};
+use searchi_service::{HybridSearch, SearchOptions, SpeechProvider, SpeechTranscription};
+use searchi_speech::{Device, SpeechWorker};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{
+    collections::BTreeMap,
+    fs,
+    io::{Read, Write},
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex},
+    time::Instant,
+};
+
+fn hash(bytes: &[u8]) -> String {
+    format!("{:x}", Sha256::digest(bytes))
+}
+fn file_hash(path: &Path) -> Result<String> {
+    let mut file = fs::File::open(path)?;
+    let mut state = Sha256::new();
+    let mut buffer = [0; 65536];
+    loop {
+        let n = file.read(&mut buffer)?;
+        if n == 0 {
+            break;
+        }
+        state.update(&buffer[..n]);
+    }
+    Ok(format!("{:x}", state.finalize()))
+}
+fn write_new(path: &Path, value: &Value) -> Result<()> {
+    let mut file = fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)?;
+    file.write_all(&serde_json::to_vec_pretty(value)?)?;
+    file.sync_all()?;
+    Ok(())
+}
+fn windows(start: u32, end: u32) -> Result<Vec<AudioWindow>> {
+    ensure!(start <= end, "Invalid source range");
+    let mut values = Vec::new();
+    let mut next = start;
+    while next < end {
+        ensure!(
+            values.len() < 4,
+            "Mini fixture exceeded bounded window count"
+        );
+        values.push(AudioWindow {
+            start_ms: next,
+            duration_ms: (end - next).min(30000),
+        });
+        if end - next <= 30000 {
+            break;
+        }
+        next = next.checked_add(28000).context("Source offset overflow")?;
+    }
+    ensure!(
+        values.len() <= 4,
+        "Mini fixture exceeded bounded window count"
+    );
+    Ok(values)
+}
+
+struct Recorder {
+    worker: SpeechWorker,
+    expected: BTreeMap<String, Vec<Value>>,
+    calls: Arc<Mutex<Vec<Value>>>,
+    directory: PathBuf,
+}
+impl SpeechProvider for Recorder {
+    fn fingerprint(&self) -> &str {
+        self.worker.fingerprint()
+    }
+    fn transcribe(&mut self, pcm: &[f32], rate: u32) -> Result<SpeechTranscription> {
+        let pcm_bytes: Vec<_> = pcm.iter().flat_map(|value| value.to_le_bytes()).collect();
+        let sha = hash(&pcm_bytes);
+        let expected = self
+            .expected
+            .get(&sha)
+            .context("Service supplied PCM not in independently verified source windows")?;
+        let started = Instant::now();
+        let result = self.worker.transcribe(pcm, rate);
+        let mut record = json!({"pcm_sha256":sha,"samples":pcm.len(),"sample_rate":rate,"expected_source_windows":expected,"native_fingerprint":self.worker.fingerprint(),"wall_ms":started.elapsed().as_secs_f64()*1000.0,"scope":"Exact accepted SpeechTranscription or returned error; native wire bytes/stderr unavailable through public client API"});
+        match &result {
+            Ok(value) => record["transcription"] = serde_json::to_value(value)?,
+            Err(error) => record["error"] = json!(format!("{error:#}")),
+        }
+        let mut calls = self
+            .calls
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Recording mutex poisoned"))?;
+        write_new(
+            &self
+                .directory
+                .join(format!("native-call-{:03}.json", calls.len() + 1)),
+            &record,
+        )?;
+        calls.push(record);
+        result
+    }
+}
+struct Assets {
+    model: PathBuf,
+    google: PathBuf,
+    speech: PathBuf,
+    base: PathBuf,
+    vad: PathBuf,
+    tools: FfmpegTools,
+}
+
+fn run_case(
+    name: &str,
+    sources: &[Value],
+    queries: &[Value],
+    assets: &Assets,
+    output: &Path,
+) -> Result<Value> {
+    let case = output.join(name);
+    fs::create_dir(&case)?;
+    let root = case.join("corpus");
+    fs::create_dir(&root)?;
+    let mut expected = BTreeMap::<String, Vec<Value>>::new();
+    let limits = MediaLimits::default();
+    let mut source_rows = Vec::new();
+    for source in sources {
+        let id = source["id"].as_str().context("ID missing")?;
+        let input = PathBuf::from(source["path"].as_str().context("Source path missing")?);
+        ensure!(
+            source["sha256"] == file_hash(&input)?,
+            "Frozen prepared source changed"
+        );
+        let path = root.join(format!("{id}.wav"));
+        fs::copy(&input, &path)?;
+        let metadata = probe(&assets.tools, &path, &limits)?;
+        let mut source_windows = Vec::new();
+        for window in windows(metadata.audio_start_ms, metadata.audio_end_ms)? {
+            let pcm = decode_audio_aligned(&assets.tools, &path, window, &limits)?;
+            ensure!(
+                pcm.len() == window.duration_ms as usize * 16,
+                "Fixture source clock coverage differs"
+            );
+            let sha = hash(
+                &pcm.iter()
+                    .flat_map(|value| value.to_le_bytes())
+                    .collect::<Vec<_>>(),
+            );
+            let record = json!({"asset":id,"start_ms":window.start_ms,"duration_ms":window.duration_ms,"samples":pcm.len(),"pcm_sha256":sha});
+            expected.entry(sha).or_default().push(record.clone());
+            source_windows.push(record);
+        }
+        source_rows.push(json!({"id":id,"path":path,"sha256":source["sha256"],"windows":source_windows,"audio_start_ms":metadata.audio_start_ms,"audio_end_ms":metadata.audio_end_ms,"piecewise_map":source.get("piecewise_map")}));
+    }
+    let db = case.join("index.sqlite");
+    let started = Instant::now();
+    let mut lexical = SearchIndex::open(&db)?;
+    let discovery = lexical.index_folder(&root)?;
+    let lexical_ms = started.elapsed().as_secs_f64() * 1000.0;
+    drop(lexical);
+    let started = Instant::now();
+    let native = SpeechWorker::load(&assets.speech, &assets.base, &assets.vad, Device::Cpu)?;
+    let speech_load_ms = started.elapsed().as_secs_f64() * 1000.0;
+    let fingerprint = native.fingerprint().to_owned();
+    let calls = Arc::new(Mutex::new(Vec::new()));
+    let recorder = Recorder {
+        worker: native,
+        expected: expected.clone(),
+        calls: calls.clone(),
+        directory: case.clone(),
+    };
+    let started = Instant::now();
+    let mut indexer = HybridSearch::open(&db, &assets.model, &assets.google, 1)?
+        .with_speech_provider(Box::new(recorder))?;
+    let google_index_load_ms = started.elapsed().as_secs_f64() * 1000.0;
+    let started = Instant::now();
+    let indexed = indexer.index_embeddings_with_media(&assets.tools, &limits)?;
+    let indexing_ms = started.elapsed().as_secs_f64() * 1000.0;
+    let google_index_worker = serde_json::to_value(indexer.worker_info())?;
+    write_new(
+        &case.join("indexed.json"),
+        &json!({"sources":source_rows,"index_report":indexed,"fingerprint":fingerprint,"worker":google_index_worker,"indexing_ms":indexing_ms}),
+    )?;
+    drop(indexer);
+    let recorded = calls
+        .lock()
+        .map_err(|_| anyhow::anyhow!("Recording mutex poisoned"))?
+        .clone();
+    let expected_calls = expected.values().map(Vec::len).sum::<usize>();
+    let complete_calls =
+        recorded.len() == expected_calls && recorded.iter().all(|row| row.get("error").is_none());
+    let connection =
+        rusqlite::Connection::open_with_flags(&db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
+    let mut statement=connection.prepare("SELECT path,number,speech_end_ms,snippet,content_hash FROM semantic_chunks WHERE generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1) AND source_type='speech' ORDER BY path,ordinal")?;
+    let chunks=statement.query_map([],|row|Ok(json!({"path":row.get::<_,String>(0)?,"start_ms":row.get::<_,u32>(1)?,"end_ms":row.get::<_,u32>(2)?,"text":row.get::<_,String>(3)?,"content_hash":row.get::<_,String>(4)?})))?.collect::<rusqlite::Result<Vec<_>>>()?;
+    let started = Instant::now();
+    let mut reader = HybridSearch::open_query(&db, &assets.model, &assets.google, 1)?;
+    let google_query_load_ms = started.elapsed().as_secs_f64() * 1000.0;
+    let mut citations = Vec::new();
+    for chunk in &chunks {
+        let excerpt = reader.speech_excerpt_matching(
+            chunk["path"].as_str().unwrap(),
+            chunk["start_ms"].as_u64().unwrap() as u32,
+            chunk["text"].as_str().unwrap(),
+            4096,
+        )?;
+        ensure!(
+            excerpt
+                .as_ref()
+                .is_some_and(|value| value.text == chunk["text"].as_str().unwrap()
+                    && value.provenance == "local_speech_recognition"),
+            "Published recognized citation missing"
+        );
+        citations.push(serde_json::to_value(excerpt)?);
+    }
+    let started = Instant::now();
+    reader.warmup()?;
+    let warmup_ms = started.elapsed().as_secs_f64() * 1000.0;
+    let mut rows = Vec::new();
+    for query in queries {
+        let response = reader.search_measured(
+            query["text"].as_str().context("Query text missing")?,
+            20,
+            &SearchOptions::default(),
+            true,
+        )?;
+        let mut hits = Vec::new();
+        for hit in response.results {
+            let id = Path::new(&hit.filename)
+                .file_stem()
+                .context("Result ID missing")?
+                .to_string_lossy()
+                .into_owned();
+            let excerpt = if hit.match_kind == "semantic_speech" {
+                let location = hit.source.as_ref().context("Speech location missing")?;
+                reader.speech_excerpt_matching(&hit.path, location.number, &hit.snippet, 4096)?
+            } else {
+                None
+            };
+            hits.push(json!({"asset":id,"result":hit,"verified_recognized_excerpt":excerpt}));
+        }
+        let row = json!({"query":query,"hits":hits,"metrics":response.metrics,"strict_point_score_eligible":false});
+        write_new(
+            &case.join(format!(
+                "query-{}.json",
+                query["id"].as_str().context("Query ID missing")?
+            )),
+            &row,
+        )?;
+        rows.push(row);
+    }
+    for source in &source_rows {
+        ensure!(
+            source["sha256"] == file_hash(Path::new(source["path"].as_str().unwrap()))?,
+            "Source changed during evaluation"
+        );
+    }
+    let report = json!({"name":name,"sources":source_rows,"lexical_report":discovery,"index_report":indexed,"expected_native_calls":expected_calls,"native_calls":recorded,"published_speech":chunks,"verified_citations":citations,"rows":rows,"native_fingerprint":fingerprint,"google_index_worker":google_index_worker,"google_query_worker":reader.worker_info(),"lexical_ms":lexical_ms,"speech_load_verify_ms":speech_load_ms,"google_index_load_verify_ms":google_index_load_ms,"indexing_ms":indexing_ms,"google_query_load_verify_ms":google_query_load_ms,"query_warmup_ms":warmup_ms,"comparison_eligible":complete_calls&&indexed.speech_failures.is_empty()&&indexed.failures.is_empty(),"strict_point_score_eligible":false});
+    write_new(&case.join("report.json"), &report)?;
+    Ok(report)
+}
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(args.len()==10,"Supply FIXTURE PREPARED GOOGLE_MODELS GOOGLE_WORKER SPEECH_WORKER BASE VAD FFMPEG FFPROBE NEW_OUTPUT_DIRECTORY");
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "Windows VM diagnostic requires CPU"
+    );
+    let fixture_path = PathBuf::from(&args[0]);
+    let fixture_raw = fs::read(&fixture_path)?;
+    let fixture: Value = serde_json::from_slice(&fixture_raw)?;
+    let input_lock: Value =
+        serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
+    let prepared_raw = fs::read(&args[1])?;
+    let prepared: Value = serde_json::from_slice(&prepared_raw)?;
+    ensure!(
+        input_lock["fixture_sha256"] == hash(&fixture_raw)
+            && input_lock["prepared_sha256"] == hash(&prepared_raw)
+            && prepared["fixture_sha256"] == hash(&fixture_raw),
+        "Frozen fixture/prepared plan differs"
+    );
+    let assets = Assets {
+        model: PathBuf::from(&args[2]),
+        google: PathBuf::from(&args[3]),
+        speech: PathBuf::from(&args[4]),
+        base: PathBuf::from(&args[5]),
+        vad: PathBuf::from(&args[6]),
+        tools: FfmpegTools::new(PathBuf::from(&args[7]), PathBuf::from(&args[8]))?,
+    };
+    ensure!(
+        prepared["ffmpeg_sha256"] == file_hash(&assets.tools.ffmpeg)?
+            && prepared["ffprobe_sha256"] == file_hash(&assets.tools.ffprobe)?,
+        "Frozen decoder differs"
+    );
+    let output = PathBuf::from(&args[9]);
+    fs::create_dir(&output)?;
+    let mut provenance = BTreeMap::new();
+    for file in [
+        "crates/searchi-speech/src/lib.rs",
+        "crates/searchi-speech/src/process.rs",
+        "crates/searchi-speech/src/inventory.rs",
+        "crates/searchi-service/src/speech.rs",
+        "crates/searchi-service/src/media.rs",
+        "crates/searchi-service/src/store.rs",
+        "tools/speech-runtime/worker.cpp",
+    ] {
+        provenance.insert(file, file_hash(Path::new(file))?);
+    }
+    write_new(
+        &output.join("preflight.json"),
+        &json!({"fixture_sha256":hash(&fixture_raw),"prepared_sha256":hash(&prepared_raw),"binary_sha256":file_hash(&std::env::current_exe()?)?,"speech_worker_sha256":file_hash(&assets.speech)?,"speech_inventory_sha256":file_hash(&assets.speech.parent().context("Runtime directory")?.join("runtime-manifest.json"))?,"base_sha256":file_hash(&assets.base)?,"vad_sha256":file_hash(&assets.vad)?,"google_worker_sha256":file_hash(&assets.google)?,"source_hashes_at_start":provenance,"speech_policy":searchi_speech::POLICY,"source_clock_policy":"ActualService speech30s overlap2s; independently decodedPCM window hashes; no strict word labels","force_cpu":true,"threads":1}),
+    )?;
+    let core = prepared["clips"].as_array().context("Core clips missing")?;
+    ensure!(core.len() == 4, "All four controls required");
+    let queries = fixture["queries"].as_array().context("Queries missing")?;
+    ensure!(queries.len() == 10, "All ten queries required");
+    let first = match run_case("core", core, queries, &assets, &output) {
+        Ok(value) => value,
+        Err(error) => {
+            write_new(
+                &output.join("failure.json"),
+                &json!({"complete":false,"case":"core","error":format!("{error:#}"),"comparison_eligible":false}),
+            )?;
+            return Err(error);
+        }
+    };
+    let seams = prepared["seam_controls"]
+        .as_array()
+        .context("Seam controls missing")?;
+    ensure!(seams.len() == 2, "Both seam sources required");
+    let second = match run_case("seams", seams, &[], &assets, &output) {
+        Ok(value) => value,
+        Err(error) => {
+            write_new(
+                &output.join("failure.json"),
+                &json!({"complete":false,"case":"seams","error":format!("{error:#}"),"comparison_eligible":false}),
+            )?;
+            return Err(error);
+        }
+    };
+    write_new(
+        &output.join("report.json"),
+        &json!({"complete":true,"core":first,"seams":second,"strict_point_score_eligible":false,"production_policy_changed":false,"scope":"Actual shared native SpeechWorker/service/Google path; independent ENDE tinycorpus; recognized outputs immutable acceptedAPIvalues, no rawwire capture; full-sourceVAD comparator not executed; seam source/window bounds only"}),
+    )
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn overlap_maps_complete_seam_tail() {
+        let value = windows(0, 40024).unwrap();
+        assert_eq!(value.len(), 2);
+        assert_eq!(value[1].start_ms, 28000);
+        assert_eq!(value[1].duration_ms, 12024);
+    }
+    #[test]
+    fn positioned_short_source_keeps_origin() {
+        let value = windows(2000, 4000).unwrap();
+        assert_eq!(value[0].start_ms, 2000);
+        assert_eq!(value[0].duration_ms, 2000);
+    }
+    #[test]
+    fn invalid_or_large_fixture_ranges_fail() {
+        assert!(windows(3, 2).is_err());
+        assert!(windows(0, 200000).is_err());
+    }
+    #[test]
+    fn existing_raw_output_is_preserved() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let path = dir.path().join("raw.json");
+        fs::write(&path, b"old")?;
+        assert!(write_new(&path, &json!({})).is_err());
+        ensure!(fs::read(path)? == b"old", "Changed report");
+        Ok(())
+    }
+}

tools/speech-eval/score_native_validation.py

@@ -0,0 +1,78 @@
+"""Recompute frozen service retrieval and negative controls; never invent point scores."""
+import argparse
+import hashlib
+import json
+from collections import Counter
+from pathlib import Path
+
+
+def score(report, fixture):
+    if report.get("complete") is not True or report.get("strict_point_score_eligible") is not False:
+        raise ValueError("Incomplete report or invented point scoring")
+    core = report["core"]
+    if {source["id"] for source in core["sources"]} != {"speech-en", "speech-de", "music-control", "silence-control"}:
+        raise ValueError("Incomplete competitor pool")
+    queries = {query["id"]: query for query in fixture["queries"]}
+    if len(core["rows"]) != len(queries) or {row["query"]["id"] for row in core["rows"]} != set(queries):
+        raise ValueError("Missing or duplicated frozen query")
+    rows = []
+    for row in core["rows"]:
+        query = row["query"]
+        if query != queries[query["id"]] or row.get("strict_point_score_eligible") is not False:
+            raise ValueError("Frozen query changed")
+        hits = row["hits"]
+        ids = [hit["asset"] for hit in hits]
+        if len(set(ids)) != len(ids) or not set(ids) <= {source["id"] for source in core["sources"]}:
+            raise ValueError("Unexpected result pool")
+        rank = next((index + 1 for index, id in enumerate(ids) if id in query["expected"]), None)
+        rows.append({"id": query["id"], "language": query["language"], "positive": bool(query["expected"]),
+                     "first_relevant_rank": rank, "returned_count": len(hits),
+                     "first_hit": None if not hits else {"asset": hits[0]["asset"], "match_kind": hits[0]["result"]["match_kind"], "snippet": hits[0]["result"]["snippet"], "source": hits[0]["result"]["source"]}})
+    positive = [row for row in rows if row["positive"]]
+    negative = [row for row in rows if not row["positive"]]
+    native_gate_counts = Counter()
+    complete_calls = True
+    for case in (core, report["seams"]):
+        expected = Counter(window["pcm_sha256"] for source in case["sources"] for window in source["windows"])
+        actual = Counter(call["pcm_sha256"] for call in case["native_calls"])
+        complete_calls &= expected == actual and all("error" not in call for call in case["native_calls"])
+    for call in core["native_calls"]:
+        ids = {window["asset"] for window in call["expected_source_windows"]}
+        if len(ids) == 1:
+            native_gate_counts[next(iter(ids))] += len(call.get("transcription", {}).get("passages", []))
+    concepts = {}
+    for asset in fixture["assets"]:
+        text = " ".join(chunk["text"] for chunk in core["published_speech"]
+                        if Path(chunk["path"]).stem == asset["id"]).casefold()
+        concepts[asset["id"]] = {keyword: keyword.casefold() in text for keyword in asset["reference_keywords"]}
+    no_false_speech = all(native_gate_counts[id] == 0 for id in ("music-control", "silence-control"))
+    eligible = complete_calls and core["comparison_eligible"] and report["seams"]["comparison_eligible"]
+    return {"comparison_eligible": eligible, "strict_point_score_eligible": False,
+            "positive_queries": len(positive), "top1": sum(row["first_relevant_rank"] == 1 for row in positive) if eligible else None,
+            "recall_at_5": sum(row["first_relevant_rank"] is not None and row["first_relevant_rank"] <= 5 for row in positive) if eligible else None,
+            "mrr": sum(1 / row["first_relevant_rank"] if row["first_relevant_rank"] else 0 for row in positive) / len(positive) if eligible else None,
+            "negative_queries": len(negative), "negative_returned_any": sum(row["returned_count"] > 0 for row in negative),
+            "native_passage_counts": dict(native_gate_counts), "negative_gate_acceptance": no_false_speech,
+            "reference_concept_substring_coverage": concepts, "rows": rows,
+            "scope": "Source-level retrieval only, substring concept diagnostic not WER; no timed publisher phrase labels or semantic confidence; retain hallucination failures"}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("report", type=Path)
+    parser.add_argument("fixture", type=Path)
+    parser.add_argument("output", type=Path)
+    args = parser.parse_args()
+    raw = args.report.read_bytes()
+    fixture_raw = args.fixture.read_bytes()
+    lock = json.loads(args.fixture.with_name("labels.lock.json").read_text())
+    if hashlib.sha256(fixture_raw).hexdigest() != lock["fixture_sha256"]:
+        raise ValueError("Frozen labels changed")
+    result = score(json.loads(raw), json.loads(fixture_raw))
+    result.update(raw_report_sha256=hashlib.sha256(raw).hexdigest(), fixture_sha256=lock["fixture_sha256"])
+    with args.output.open("x", encoding="utf-8") as output:
+        json.dump(result, output, ensure_ascii=False, indent=2)
+
+
+if __name__ == "__main__":
+    main()

tools/speech-eval/test_score_native_validation.py

@@ -0,0 +1,50 @@
+import copy
+import unittest
+
+from score_native_validation import score
+
+
+class ScoringTests(unittest.TestCase):
+    def inputs(self):
+        fixture = {"assets": [{"id": "speech-en", "reference_keywords": ["equal"]}, {"id": "speech-de", "reference_keywords": ["gleich"]}],
+                   "queries": [{"id": "positive", "language": "en", "text": "equal dignity", "expected": ["speech-en", "speech-de"]}, {"id": "absent", "language": "en", "text": "salary", "expected": []}]}
+        sources = [{"id": id, "windows": [{"pcm_sha256": id}]} for id in ("speech-en", "speech-de", "music-control", "silence-control")]
+        calls = [{"pcm_sha256": source["id"], "expected_source_windows": [{"asset": source["id"]}], "transcription": {"passages": []}} for source in sources]
+        core = {"sources": sources, "native_calls": calls, "comparison_eligible": True, "published_speech": [], "rows": []}
+        for query in fixture["queries"]:
+            core["rows"].append({"query": query, "strict_point_score_eligible": False,
+                                 "hits": [{"asset": "speech-de", "result": {"match_kind": "semantic_speech", "snippet": "gleich", "source": {"kind": "timestamp_ms", "number": 5000}}}]})
+        return {"complete": True, "strict_point_score_eligible": False, "core": core, "seams": {"sources": [], "native_calls": [], "comparison_eligible": True}}, fixture
+
+    def test_no_answer_return_is_failure_not_positive_recall(self):
+        report, fixture = self.inputs()
+        result = score(report, fixture)
+        self.assertEqual(result["top1"], 1)
+        self.assertEqual(result["negative_returned_any"], 1)
+        self.assertFalse(result["strict_point_score_eligible"])
+
+    def test_dropped_native_window_invalidates_comparison(self):
+        report, fixture = self.inputs()
+        report["core"]["native_calls"].pop()
+        result = score(report, fixture)
+        self.assertFalse(result["comparison_eligible"])
+        self.assertIsNone(result["top1"])
+        self.assertIsNone(result["mrr"])
+
+    def test_silence_hallucination_is_retained_failure(self):
+        report, fixture = self.inputs()
+        report["core"]["native_calls"][-1]["transcription"]["passages"] = [{"text": "fake speech"}]
+        self.assertFalse(score(report, fixture)["negative_gate_acceptance"])
+
+    def test_frozen_query_mutation_is_rejected(self):
+        report, fixture = self.inputs()
+        report = copy.deepcopy(report)
+        report["core"]["rows"][0]["query"]["text"] = "changed"
+        with self.assertRaises(ValueError):
+            score(report, fixture)
+
+    def test_duplicate_query_is_rejected(self):
+        report, fixture = self.inputs()
+        report["core"]["rows"][1] = report["core"]["rows"][0]
+        with self.assertRaises(ValueError):
+            score(report, fixture)