Validate finer speech passages independently before Google ranking

tools/speech-rank/README.md

@@ -57,3 +57,21 @@ ASR segment approximations, potentially affected by 30-second recognition
 boundaries; they are **not word alignment**. A token window retains its containing
 segment's same interval. The existing eight frozen NASA queries constitute a tiny
 diagnostic, not held-out calibration, general media quality or production proof.
+
+The optional frozen `ordinary-token-sentences-v1` input retains those original
+segments and supplies full Whisper JSON. The ranker independently reconstructs
+exact recognized sentences from ordinary token times, excludes structural tokens
+from passage text, and verifies the declared policy and Python contract hashes.
+Changed text, missing/out-of-range/unordered timestamps, forged normalized
+passages, omitted windows or partial provenance are rejected. All five sources
+remain competitors. Real alignment failures are retained as excluded windows;
+such a report cannot receive an accepted quality score. The old ungated/gated
+policies are unchanged. Token times remain approximate recognized-speech
+coordinates; this experiment does not make them verified quotations or add
+production transcription.
+
+After the complete base token-sentence ASR run and coordinator CPU-slot grant,
+use the existing executable command with input
+`.searchi/speech/base-token-sentences-v1.json` and a new output filename, then the
+same frozen eight-query scorer. Do not overwrite old results or select passages
+with relevance labels.

tools/speech-rank/src/alignment.rs

@@ -0,0 +1,190 @@
+//! Independent reconstruction of the frozen ordinary token-time experiment.
+use super::{number, parsed_segments};
+use anyhow::{ensure, Context, Result};
+use serde_json::{json, Value};
+
+pub const POLICY: &str =
+    "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels";
+pub const POLICY_SHA: &str = "5bc603e792a3abcc2370d4d1f7a13f538f808fb3b959b506a16d5823d3d7a126";
+pub const CONTRACT_SHA: &str = "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520";
+
+pub fn enabled(asr: &Value) -> Result<bool> {
+    if asr.get("alignment_policy").is_none() {
+        ensure!(
+            asr.get("alignment_contract_sha256").is_none()
+                && asr.get("alignment_policy_sha256").is_none(),
+            "Partial alignment provenance"
+        );
+        return Ok(false);
+    }
+    ensure!(
+        asr["alignment_policy"] == POLICY
+            && asr["alignment_policy_sha256"] == POLICY_SHA
+            && asr["alignment_contract_sha256"] == CONTRACT_SHA
+            && asr["candidate"] == "base"
+            && asr["worker_sha256"]
+                == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8"
+            && asr["policy"] == super::POLICY,
+        "Unknown token alignment provenance"
+    );
+    Ok(true)
+}
+
+fn boundary(text: &str) -> bool {
+    // Python's frozen regex `$` also matches before a final newline.
+    let text = text.strip_suffix('\n').unwrap_or(text);
+    text.trim_end_matches(['"', '\u{201d}', '\u{2019}', '\'', ')', ']'])
+        .ends_with(['.', '!', '?', '。', '！', '？'])
+}
+
+pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
+    let originals = parsed_segments(raw, offset, duration)?;
+    let rows = raw["transcription"]
+        .as_array()
+        .context("Missing transcription")?;
+    let mut result = Vec::new();
+    let mut original_index = 0;
+    for row in rows {
+        let text = row["text"].as_str().context("Missing original text")?;
+        if text.trim().is_empty() {
+            continue;
+        }
+        let original = &originals[original_index];
+        let tokens = row["tokens"].as_array().context("Missing token times")?;
+        ensure!(
+            !tokens.is_empty() && tokens.len() <= 4096,
+            "Unbounded token times"
+        );
+        let mut exact = String::new();
+        let mut phrase = String::new();
+        let mut phrase_start = None;
+        let mut phrase_end = 0;
+        let mut previous = 0;
+        for token in tokens {
+            let id = number(&token["id"])?;
+            let token_text = token["text"].as_str().context("Invalid token text")?;
+            if id >= 50257 {
+                ensure!(
+                    token_text.starts_with("[_") && token_text.ends_with(']'),
+                    "Invalid special token"
+                );
+                continue;
+            }
+            let start = number(&token["offsets"]["from"])?;
+            let end = number(&token["offsets"]["to"])?;
+            ensure!(
+                start >= previous
+                    && start <= end
+                    && end <= duration
+                    && start >= number(&row["offsets"]["from"])?
+                    && end <= number(&row["offsets"]["to"])?,
+                "Invalid ordinary token time"
+            );
+            previous = start;
+            exact.push_str(token_text);
+            ensure!(exact.len() <= 4096, "Oversized token text");
+            phrase_start.get_or_insert(start);
+            phrase_end = end;
+            phrase.push_str(token_text);
+            if boundary(token_text) {
+                append(
+                    &mut result,
+                    &phrase,
+                    phrase_start,
+                    phrase_end,
+                    offset,
+                    original_index,
+                    original,
+                )?;
+                phrase.clear();
+                phrase_start = None;
+            }
+        }
+        ensure!(
+            exact == text,
+            "Token text differs from original recognized segment"
+        );
+        append(
+            &mut result,
+            &phrase,
+            phrase_start,
+            phrase_end,
+            offset,
+            original_index,
+            original,
+        )?;
+        original_index += 1;
+    }
+    ensure!(result.len() <= 10000, "Oversized aligned passages");
+    Ok(result)
+}
+
+fn append(
+    result: &mut Vec<Value>,
+    text: &str,
+    start: Option<u64>,
+    end: u64,
+    offset: u64,
+    index: usize,
+    original: &Value,
+) -> Result<()> {
+    if !text.trim().is_empty() {
+        result.push(json!({"start_ms":offset.checked_add(start.context("Missing phrase start")?).context("Timestamp overflow")?,
+            "end_ms":offset.checked_add(end).context("Timestamp overflow")?, "text":text,
+            "original_segment_index":index,"original_start_ms":original["start_ms"],"original_end_ms":original["end_ms"]}));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    fn raw() -> Value {
+        json!({"transcription":[{"text":" Hello. Guten Tag!", "offsets":{"from":0,"to":2000},
+          "tokens":[{"id":50364,"text":"[_BEG_]"},
+            {"id":100,"text":" Hello.","offsets":{"from":100,"to":900}},
+            {"id":101,"text":" Guten","offsets":{"from":1100,"to":1500}},
+            {"id":102,"text":" Tag!","offsets":{"from":1500,"to":1900}}]}]})
+    }
+    #[test]
+    fn reconstructs_exact_sentences_and_absolute_source_coordinates() {
+        let result = reconstruct(&raw(), 30000, 2000).unwrap();
+        assert_eq!(result.len(), 2);
+        assert_eq!(
+            result[0],
+            json!({"start_ms":30100,"end_ms":30900,"text":" Hello.","original_segment_index":0,"original_start_ms":30000,"original_end_ms":32000})
+        );
+        assert_eq!(result[1]["text"], " Guten Tag!");
+        assert_eq!(result[1]["start_ms"], 31100);
+        assert!(reconstruct(&raw(), u64::MAX, 2000).is_err());
+    }
+    #[test]
+    fn rejects_missing_times_outside_bounds_nonmonotonic_and_changed_text() {
+        for replacement in [
+            json!({"id":100,"text":" Hello."}),
+            json!({"id":100,"text":" Hello.","offsets":{"from":0,"to":3000}}),
+            json!({"id":100,"text":" changed","offsets":{"from":100,"to":900}}),
+            json!({"id":50364,"text":"real words"}),
+        ] {
+            let mut changed = raw();
+            changed["transcription"][0]["tokens"][1] = replacement;
+            assert!(reconstruct(&changed, 0, 2000).is_err());
+        }
+        let mut changed = raw();
+        changed["transcription"][0]["tokens"][3]["offsets"]["from"] = json!(1000);
+        assert!(reconstruct(&changed, 0, 2000).is_err());
+    }
+    #[test]
+    fn provenance_cannot_be_partial_unknown_or_small_model() {
+        assert!(!enabled(&json!({})).unwrap());
+        assert!(enabled(&json!({"alignment_contract_sha256":CONTRACT_SHA})).is_err());
+        let report = json!({"alignment_policy":POLICY,"alignment_policy_sha256":POLICY_SHA,
+            "alignment_contract_sha256":CONTRACT_SHA,"candidate":"base",
+            "worker_sha256":"12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8",
+            "policy":super::super::POLICY});
+        assert!(enabled(&report).unwrap());
+        let mut changed = report;
+        changed["candidate"] = json!("small");
+        assert!(enabled(&changed).is_err());
+    }
+}

tools/speech-rank/src/main.rs

@@ -13,6 +13,7 @@ use std::{
 
 const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
 const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"];
+mod alignment;
 mod gated;
 const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times";
 
@@ -105,6 +106,7 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
     ] {
         ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance");
     }
+    let token_alignment = alignment::enabled(asr)?;
     let vad = gated::verified_report(asr)?;
     let rows = asr["rows"].as_array().context("Missing ASR rows")?;
     ensure!(rows.len() == 5, "All five competing assets required");
@@ -187,6 +189,31 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
                 window["segments"] == json!(actual),
                 "Normalized ASR segments differ from actual output"
             );
+            let actual = if token_alignment {
+                let rebuilt =
+                    alignment::reconstruct(&window["actual_whisper_json"], offset, duration);
+                if let Some(error) = window.get("alignment_validation_error") {
+                    ensure!(
+                        error.as_str().is_some_and(|e| !e.is_empty()) && rebuilt.is_err(),
+                        "Valid aligned text must not be excluded"
+                    );
+                    excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error}));
+                    continue;
+                }
+                let rebuilt = rebuilt?;
+                ensure!(
+                    window["aligned_segments"] == json!(rebuilt),
+                    "Normalized token passages differ from actual output"
+                );
+                rebuilt
+            } else {
+                ensure!(
+                    window.get("aligned_segments").is_none()
+                        && window.get("alignment_validation_error").is_none(),
+                    "Alignment without provenance"
+                );
+                actual
+            };
             let title = Path::new(asset["path"].as_str().context("Missing source filename")?)
                 .file_name()
                 .context("Missing filename")?
@@ -319,6 +346,12 @@ fn main() -> Result<()> {
     report["excluded_window_count"] = json!(excluded.len());
     report["quality_score_eligible"] = json!(excluded.is_empty());
     report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
+    if alignment::enabled(&asr)? {
+        report["scope"] = json!("Diagnostic ordinary token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate token intervals, no labels in ranking; not production integration.");
+        report["alignment_policy"] = json!(alignment::POLICY);
+        report["alignment_policy_sha256"] = json!(alignment::POLICY_SHA);
+        report["alignment_contract_sha256"] = json!(alignment::CONTRACT_SHA);
+    }
     report["phase"] = json!("document_encoding");
     checkpoint(&output, &report)?;
     let start = Instant::now();
@@ -431,6 +464,39 @@ mod tests {
         .is_err());
     }
 
+    #[test]
+    fn aligned_pool_preserves_all_competitors_and_marks_real_failures_ineligible() {
+        let hash = "a".repeat(64);
+        let assets: Vec<_> = ASSETS
+            .iter()
+            .map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.wav")}))
+            .collect();
+        let raw = json!({"transcription":[{"text":" First. Second!","offsets":{"from":0,"to":1000},"tokens":[
+            {"id":100,"text":" First.","offsets":{"from":100,"to":400}},
+            {"id":101,"text":" Second!","offsets":{"from":600,"to":900}}]}]});
+        let rows:Vec<_> = ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap(),"aligned_segments":alignment::reconstruct(&raw,1000,1000).unwrap()}]})).collect();
+        let mut report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":"12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8","harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows,"alignment_policy":alignment::POLICY,"alignment_policy_sha256":alignment::POLICY_SHA,"alignment_contract_sha256":alignment::CONTRACT_SHA});
+        let fixture = json!({"assets":assets});
+        assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 10);
+        let mut forged = report.clone();
+        forged["rows"][0]["windows"][0]["aligned_segments"][0]["start_ms"] = json!(1500);
+        assert!(eligible(&forged, &fixture).is_err());
+        let window = &mut report["rows"][0]["windows"][0];
+        window["actual_whisper_json"]["transcription"][0]["tokens"][0]
+            .as_object_mut()
+            .unwrap()
+            .remove("offsets");
+        window["alignment_validation_error"] = json!("Missing token time");
+        window.as_object_mut().unwrap().remove("aligned_segments");
+        report["quality_failed"] = json!(true);
+        report["validation_failures"] = json!([{"error":"Missing token time"}]);
+        let (pool, excluded) = eligible(&report, &fixture).unwrap();
+        assert_eq!(pool.len(), 8);
+        assert_eq!(excluded.len(), 1);
+        // Main derives quality_score_eligible solely from excluded.is_empty().
+        assert!(!excluded.is_empty());
+    }
+
     #[test]
     fn source_identity_coverage_and_recognized_text_cannot_be_forged() {
         let hash = "a".repeat(64);