Validate gated Google retrieval and record unchanged localization quality

tools/speech-rank/README.md

@@ -19,6 +19,17 @@ ranks are retained; at most the first 100 segment hits per query are exported fo
 the existing scorer. Ties use stable source order. No confidence threshold or
 oracle matching is applied.
 
+The additional frozen `gated-vad-v1` diagnostic policy accepts only the exact
+VAD report SHA-256 `093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df`.
+The ASR report embeds its original UTF-8 bytes and parsed document; both must
+agree. Source probes, decoder hashes and the pinned base Whisper worker must
+match. The ranker independently rebuilds every contiguous window from the frozen
+bounded source-sample intersections, retaining interval identity and explicit
+sub-millisecond tail remainder. Omitted intervals, changed windows or claimed
+no-speech status without frozen empty intervals are rejected before inference.
+All five sources remain explicit competitors; no relevance label selects which
+source receives ASR. Existing ungated reports still require full audio coverage.
+
 ```powershell
 cargo test --manifest-path tools/speech-rank/Cargo.toml --locked
 cargo build --release --manifest-path tools/speech-rank/Cargo.toml --locked

tools/speech-rank/src/gated.rs

@@ -0,0 +1,193 @@
+//! Gated diagnostics are anchored to exact frozen VAD bytes, never claimed windows.
+use super::*;
+pub const POLICY: &str = "gated-vad-v1;source-intersections;integer-ms-tail-remainder-recorded;contiguous30s;independent-process;auto-language;segment-times";
+const VAD_SHA: &str = "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df";
+
+pub fn verified_report(asr: &Value) -> Result<Option<Value>> {
+    if asr["policy"] != POLICY {
+        return Ok(None);
+    }
+    let bytes = asr["vad_report_utf8"]
+        .as_str()
+        .context("Missing original VAD bytes")?
+        .as_bytes();
+    ensure!(
+        bytes.len() <= 16 * 1024 * 1024
+            && format!("{:x}", Sha256::digest(bytes)) == VAD_SHA
+            && asr["vad_report_sha256"] == VAD_SHA,
+        "Frozen VAD bytes changed"
+    );
+    let vad: Value = serde_json::from_slice(bytes)?;
+    ensure!(
+        asr["actual_vad_report"] == vad,
+        "VAD parsed provenance differs from original bytes"
+    );
+    ensure!(
+        asr["candidate"] == "base"
+            && asr["worker_sha256"]
+                == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8",
+        "Unknown gated ASR worker/model"
+    );
+    for name in ["ffmpeg_sha256", "ffprobe_sha256"] {
+        ensure!(asr[name] == vad[name], "VAD and ASR decoder differ");
+    }
+    Ok(Some(vad))
+}
+
+pub fn planned_windows(vad: &Value, row: &Value, asset: &Value) -> Result<Vec<Value>> {
+    let detected = vad["rows"]
+        .as_array()
+        .context("Missing frozen VAD rows")?
+        .iter()
+        .find(|v| v["asset"] == row["asset"])
+        .context("Missing VAD competitor")?;
+    ensure!(
+        detected["source_sha256"] == asset["sha256"] && row["actual_probe"] == detected["probe"],
+        "VAD source/probe differs"
+    );
+    let audio = detected["probe"]["streams"]
+        .as_array()
+        .context("Missing audio probe")?
+        .iter()
+        .find(|v| v["codec_type"] == "audio")
+        .context("Missing primary audio")?;
+    let start: f64 = audio["start_time"].as_str().unwrap_or("0").parse()?;
+    let duration: f64 = audio["duration"]
+        .as_str()
+        .context("Missing audio duration")?
+        .parse()?;
+    ensure!(
+        start.is_finite()
+            && start >= 0.0
+            && duration.is_finite()
+            && duration > 0.0
+            && duration <= 600.0,
+        "Invalid source timeline"
+    );
+    let origin = (start * 1000.0).ceil() as u64;
+    let end = ((start + duration) * 1000.0).ceil() as u64;
+    ensure!(
+        row["audio_start_ms"] == origin
+            && row["audio_end_ms"] == end
+            && row["audio_stream_index"] == audio["index"],
+        "ASR timeline differs from frozen probe"
+    );
+    let samples = number(&detected["native"]["samples"])?;
+    let intervals = detected["intervals"]
+        .as_array()
+        .context("Missing VAD intervals")?;
+    let mut result = Vec::new();
+    let mut previous = 0;
+    for (index, interval) in intervals.iter().enumerate() {
+        let raw_start = number(&interval["raw_cs"][0])?
+            .checked_mul(160)
+            .context("Interval overflow")?;
+        let raw_end = number(&interval["raw_cs"][1])?
+            .checked_mul(160)
+            .context("Interval overflow")?;
+        let first = raw_start.min(samples);
+        let last = raw_end.min(samples);
+        ensure!(
+            raw_end > raw_start
+                && first >= previous
+                && interval["source_samples"] == json!([first, last])
+                && interval["tail_padding_samples"] == raw_end.saturating_sub(samples),
+            "Invalid bounded VAD intersection"
+        );
+        previous = last;
+        let mut offset = origin + first / 16;
+        let end = origin + last / 16;
+        while offset < end {
+            let duration = (end - offset).min(30000);
+            result.push(json!({"offset_ms":offset,"duration_ms":duration,"vad_interval_index":index,"vad_source_intersection":interval,"unrepresented_tail_samples":last%16}));
+            ensure!(result.len() <= 20, "Too many gated windows");
+            offset += duration;
+        }
+    }
+    ensure!(
+        row["status"]
+            == if result.is_empty() {
+                "no_speech"
+            } else {
+                "speech"
+            }
+            && row["asr_not_run"] == result.is_empty(),
+        "Missing explicit speech/no-speech evidence"
+    );
+    Ok(result)
+}
+
+pub fn matches_plan(window: &Value, planned: &Value) -> bool {
+    [
+        "offset_ms",
+        "duration_ms",
+        "vad_interval_index",
+        "vad_source_intersection",
+        "unrepresented_tail_samples",
+    ]
+    .iter()
+    .all(|key| window[key] == planned[key])
+}
+
+pub fn verify_windows(windows: &[Value], planned: &[Value]) -> Result<()> {
+    ensure!(
+        windows.len() == planned.len()
+            && windows
+                .iter()
+                .zip(planned)
+                .all(|(window, plan)| matches_plan(window, plan)),
+        "Incomplete or changed VAD interval windows"
+    );
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    fn source(intervals: Value) -> (Value, Value, Value) {
+        let probe = json!({"streams":[{"codec_type":"audio","index":2,"start_time":"0.125","duration":"65.0"}]});
+        let vad = json!({"rows":[{"asset":"clip04","source_sha256":"sha","probe":probe,"native":{"samples":1040000},"intervals":intervals}]});
+        let row = json!({"asset":"clip04","actual_probe":probe,"audio_start_ms":125,"audio_end_ms":65125,"audio_stream_index":2,"status":"speech","asr_not_run":false});
+        (vad, row, json!({"sha256":"sha"}))
+    }
+    #[test]
+    fn windows_reconstruct_original_intervals_without_merging() {
+        let intervals = json!([
+            {"raw_cs":[0,3100],"source_samples":[0,496000],"tail_padding_samples":0},
+            {"raw_cs":[4000,4200],"source_samples":[640000,672000],"tail_padding_samples":0}
+        ]);
+        let (vad, row, asset) = source(intervals);
+        let plan = planned_windows(&vad, &row, &asset).unwrap();
+        assert_eq!(plan.len(), 3);
+        assert_eq!(plan[0]["offset_ms"], 125);
+        assert_eq!(plan[1]["offset_ms"], 30125);
+        assert_eq!(plan[1]["duration_ms"], 1000);
+        assert_eq!(plan[2]["offset_ms"], 40125);
+        assert!(verify_windows(&plan, &plan).is_ok());
+        assert!(verify_windows(&plan[..2], &plan).is_err());
+        let mut omitted = plan[2].clone();
+        omitted["vad_interval_index"] = json!(0);
+        assert!(!matches_plan(&omitted, &plan[2]));
+        let mut changed = vad.clone();
+        changed["rows"][0]["intervals"][0]["source_samples"][1] = json!(480000);
+        assert!(planned_windows(&changed, &row, &asset).is_err());
+    }
+    #[test]
+    fn no_speech_requires_empty_frozen_intervals_and_explicit_status() {
+        let (vad, mut row, asset) = source(json!([]));
+        assert!(planned_windows(&vad, &row, &asset).is_err());
+        row["status"] = json!("no_speech");
+        row["asr_not_run"] = json!(true);
+        assert!(planned_windows(&vad, &row, &asset).unwrap().is_empty());
+        row["audio_start_ms"] = json!(0);
+        assert!(planned_windows(&vad, &row, &asset).is_err());
+    }
+    #[test]
+    fn arbitrary_vad_claim_cannot_authorize_candidate_exclusion() {
+        let report = json!({"policy":POLICY,"vad_report_sha256":VAD_SHA,"vad_report_utf8":"{}","actual_vad_report":{}});
+        assert!(verified_report(&report).is_err());
+        assert!(verified_report(&json!({"policy":super::super::POLICY}))
+            .unwrap()
+            .is_none());
+    }
+}

tools/speech-rank/src/main.rs

@@ -13,6 +13,7 @@ use std::{
 
 const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
 const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"];
+mod gated;
 const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times";
 
 fn digest(path: &Path) -> Result<String> {
@@ -90,7 +91,7 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
     ensure!(
         asr["complete"] == true
             && asr["fixture_sha256"] == FIXTURE_SHA
-            && asr["policy"] == POLICY
+            && matches!(asr["policy"].as_str(), Some(POLICY) | Some(gated::POLICY))
             && asr["backend"] == "forced CPU"
             && asr["threads"] == 1,
         "Incomplete, stale, or unrecognized ASR report"
@@ -104,6 +105,7 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
     ] {
         ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance");
     }
+    let vad = gated::verified_report(asr)?;
     let rows = asr["rows"].as_array().context("Missing ASR rows")?;
     ensure!(rows.len() == 5, "All five competing assets required");
     let mut seen = BTreeSet::new();
@@ -133,12 +135,23 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
         );
         let windows = row["windows"].as_array().context("Missing windows")?;
         ensure!(windows.len() <= 20, "Oversized window list");
+        let planned = vad
+            .as_ref()
+            .map(|vad| gated::planned_windows(vad, row, asset))
+            .transpose()?;
+        if let Some(planned) = &planned {
+            gated::verify_windows(windows, planned)?;
+        }
         let mut next = start;
         for (window_index, window) in windows.iter().enumerate() {
             let offset = number(&window["offset_ms"])?;
             let duration = number(&window["duration_ms"])?;
             ensure!(
-                offset == next && duration == (end - offset).min(30000) && duration > 0,
+                if let Some(planned) = &planned {
+                    gated::matches_plan(window, &planned[window_index])
+                } else {
+                    offset == next && duration == (end - offset).min(30000) && duration > 0
+                },
                 "Missing, overlapping or truncated source windows"
             );
             next = offset.checked_add(duration).context("Window overflow")?;
@@ -183,7 +196,10 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
                 ensure!(segments.len() <= 5000, "Too many recognized segments");
             }
         }
-        ensure!(next == end, "Incomplete source-window coverage");
+        ensure!(
+            planned.is_some() || next == end,
+            "Incomplete source-window coverage"
+        );
     }
     ensure!(
         asr["quality_failed"] == !excluded.is_empty(),
@@ -197,6 +213,12 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
             == excluded.len(),
         "ASR validation failure count inconsistent"
     );
+    if vad.is_some() {
+        ensure!(
+            asr["comparison_eligible"] == excluded.is_empty(),
+            "Gated comparison status inconsistent"
+        );
+    }
     Ok((segments, excluded))
 }