diff --git a/tools/speech-rank/README.md b/tools/speech-rank/README.md index 06eaae7..4aefa0e 100644 --- a/tools/speech-rank/README.md +++ b/tools/speech-rank/README.md @@ -57,3 +57,21 @@ ASR segment approximations, potentially affected by 30-second recognition boundaries; they are **not word alignment**. A token window retains its containing segment's same interval. The existing eight frozen NASA queries constitute a tiny diagnostic, not held-out calibration, general media quality or production proof. + +The optional frozen `ordinary-token-sentences-v1` input retains those original +segments and supplies full Whisper JSON. The ranker independently reconstructs +exact recognized sentences from ordinary token times, excludes structural tokens +from passage text, and verifies the declared policy and Python contract hashes. +Changed text, missing/out-of-range/unordered timestamps, forged normalized +passages, omitted windows or partial provenance are rejected. All five sources +remain competitors. Real alignment failures are retained as excluded windows; +such a report cannot receive an accepted quality score. The old ungated/gated +policies are unchanged. Token times remain approximate recognized-speech +coordinates; this experiment does not make them verified quotations or add +production transcription. + +After the complete base token-sentence ASR run and coordinator CPU-slot grant, +use the existing executable command with input +`.searchi/speech/base-token-sentences-v1.json` and a new output filename, then the +same frozen eight-query scorer. Do not overwrite old results or select passages +with relevance labels. diff --git a/tools/speech-rank/src/alignment.rs b/tools/speech-rank/src/alignment.rs new file mode 100644 index 0000000..29405a7 --- /dev/null +++ b/tools/speech-rank/src/alignment.rs @@ -0,0 +1,190 @@ +//! Independent reconstruction of the frozen ordinary token-time experiment. +use super::{number, parsed_segments}; +use anyhow::{ensure, Context, Result}; +use serde_json::{json, Value}; + +pub const POLICY: &str = + "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels"; +pub const POLICY_SHA: &str = "5bc603e792a3abcc2370d4d1f7a13f538f808fb3b959b506a16d5823d3d7a126"; +pub const CONTRACT_SHA: &str = "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520"; + +pub fn enabled(asr: &Value) -> Result { + if asr.get("alignment_policy").is_none() { + ensure!( + asr.get("alignment_contract_sha256").is_none() + && asr.get("alignment_policy_sha256").is_none(), + "Partial alignment provenance" + ); + return Ok(false); + } + ensure!( + asr["alignment_policy"] == POLICY + && asr["alignment_policy_sha256"] == POLICY_SHA + && asr["alignment_contract_sha256"] == CONTRACT_SHA + && asr["candidate"] == "base" + && asr["worker_sha256"] + == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8" + && asr["policy"] == super::POLICY, + "Unknown token alignment provenance" + ); + Ok(true) +} + +fn boundary(text: &str) -> bool { + // Python's frozen regex `$` also matches before a final newline. + let text = text.strip_suffix('\n').unwrap_or(text); + text.trim_end_matches(['"', '\u{201d}', '\u{2019}', '\'', ')', ']']) + .ends_with(['.', '!', '?', '。', '！', '？']) +} + +pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result> { + let originals = parsed_segments(raw, offset, duration)?; + let rows = raw["transcription"] + .as_array() + .context("Missing transcription")?; + let mut result = Vec::new(); + let mut original_index = 0; + for row in rows { + let text = row["text"].as_str().context("Missing original text")?; + if text.trim().is_empty() { + continue; + } + let original = &originals[original_index]; + let tokens = row["tokens"].as_array().context("Missing token times")?; + ensure!( + !tokens.is_empty() && tokens.len() <= 4096, + "Unbounded token times" + ); + let mut exact = String::new(); + let mut phrase = String::new(); + let mut phrase_start = None; + let mut phrase_end = 0; + let mut previous = 0; + for token in tokens { + let id = number(&token["id"])?; + let token_text = token["text"].as_str().context("Invalid token text")?; + if id >= 50257 { + ensure!( + token_text.starts_with("[_") && token_text.ends_with(']'), + "Invalid special token" + ); + continue; + } + let start = number(&token["offsets"]["from"])?; + let end = number(&token["offsets"]["to"])?; + ensure!( + start >= previous + && start <= end + && end <= duration + && start >= number(&row["offsets"]["from"])? + && end <= number(&row["offsets"]["to"])?, + "Invalid ordinary token time" + ); + previous = start; + exact.push_str(token_text); + ensure!(exact.len() <= 4096, "Oversized token text"); + phrase_start.get_or_insert(start); + phrase_end = end; + phrase.push_str(token_text); + if boundary(token_text) { + append( + &mut result, + &phrase, + phrase_start, + phrase_end, + offset, + original_index, + original, + )?; + phrase.clear(); + phrase_start = None; + } + } + ensure!( + exact == text, + "Token text differs from original recognized segment" + ); + append( + &mut result, + &phrase, + phrase_start, + phrase_end, + offset, + original_index, + original, + )?; + original_index += 1; + } + ensure!(result.len() <= 10000, "Oversized aligned passages"); + Ok(result) +} + +fn append( + result: &mut Vec, + text: &str, + start: Option, + end: u64, + offset: u64, + index: usize, + original: &Value, +) -> Result<()> { + if !text.trim().is_empty() { + result.push(json!({"start_ms":offset.checked_add(start.context("Missing phrase start")?).context("Timestamp overflow")?, + "end_ms":offset.checked_add(end).context("Timestamp overflow")?, "text":text, + "original_segment_index":index,"original_start_ms":original["start_ms"],"original_end_ms":original["end_ms"]})); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn raw() -> Value { + json!({"transcription":[{"text":" Hello. Guten Tag!", "offsets":{"from":0,"to":2000}, + "tokens":[{"id":50364,"text":"[_BEG_]"}, + {"id":100,"text":" Hello.","offsets":{"from":100,"to":900}}, + {"id":101,"text":" Guten","offsets":{"from":1100,"to":1500}}, + {"id":102,"text":" Tag!","offsets":{"from":1500,"to":1900}}]}]}) + } + #[test] + fn reconstructs_exact_sentences_and_absolute_source_coordinates() { + let result = reconstruct(&raw(), 30000, 2000).unwrap(); + assert_eq!(result.len(), 2); + assert_eq!( + result[0], + json!({"start_ms":30100,"end_ms":30900,"text":" Hello.","original_segment_index":0,"original_start_ms":30000,"original_end_ms":32000}) + ); + assert_eq!(result[1]["text"], " Guten Tag!"); + assert_eq!(result[1]["start_ms"], 31100); + assert!(reconstruct(&raw(), u64::MAX, 2000).is_err()); + } + #[test] + fn rejects_missing_times_outside_bounds_nonmonotonic_and_changed_text() { + for replacement in [ + json!({"id":100,"text":" Hello."}), + json!({"id":100,"text":" Hello.","offsets":{"from":0,"to":3000}}), + json!({"id":100,"text":" changed","offsets":{"from":100,"to":900}}), + json!({"id":50364,"text":"real words"}), + ] { + let mut changed = raw(); + changed["transcription"][0]["tokens"][1] = replacement; + assert!(reconstruct(&changed, 0, 2000).is_err()); + } + let mut changed = raw(); + changed["transcription"][0]["tokens"][3]["offsets"]["from"] = json!(1000); + assert!(reconstruct(&changed, 0, 2000).is_err()); + } + #[test] + fn provenance_cannot_be_partial_unknown_or_small_model() { + assert!(!enabled(&json!({})).unwrap()); + assert!(enabled(&json!({"alignment_contract_sha256":CONTRACT_SHA})).is_err()); + let report = json!({"alignment_policy":POLICY,"alignment_policy_sha256":POLICY_SHA, + "alignment_contract_sha256":CONTRACT_SHA,"candidate":"base", + "worker_sha256":"12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8", + "policy":super::super::POLICY}); + assert!(enabled(&report).unwrap()); + let mut changed = report; + changed["candidate"] = json!("small"); + assert!(enabled(&changed).is_err()); + } +} diff --git a/tools/speech-rank/src/main.rs b/tools/speech-rank/src/main.rs index eb3577c..1f3fc12 100644 --- a/tools/speech-rank/src/main.rs +++ b/tools/speech-rank/src/main.rs @@ -13,6 +13,7 @@ use std::{ const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393"; const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"]; +mod alignment; mod gated; const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times"; @@ -105,6 +106,7 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { ] { ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance"); } + let token_alignment = alignment::enabled(asr)?; let vad = gated::verified_report(asr)?; let rows = asr["rows"].as_array().context("Missing ASR rows")?; ensure!(rows.len() == 5, "All five competing assets required"); @@ -187,6 +189,31 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { window["segments"] == json!(actual), "Normalized ASR segments differ from actual output" ); + let actual = if token_alignment { + let rebuilt = + alignment::reconstruct(&window["actual_whisper_json"], offset, duration); + if let Some(error) = window.get("alignment_validation_error") { + ensure!( + error.as_str().is_some_and(|e| !e.is_empty()) && rebuilt.is_err(), + "Valid aligned text must not be excluded" + ); + excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error})); + continue; + } + let rebuilt = rebuilt?; + ensure!( + window["aligned_segments"] == json!(rebuilt), + "Normalized token passages differ from actual output" + ); + rebuilt + } else { + ensure!( + window.get("aligned_segments").is_none() + && window.get("alignment_validation_error").is_none(), + "Alignment without provenance" + ); + actual + }; let title = Path::new(asset["path"].as_str().context("Missing source filename")?) .file_name() .context("Missing filename")? @@ -319,6 +346,12 @@ fn main() -> Result<()> { report["excluded_window_count"] = json!(excluded.len()); report["quality_score_eligible"] = json!(excluded.is_empty()); report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration."); + if alignment::enabled(&asr)? { + report["scope"] = json!("Diagnostic ordinary token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate token intervals, no labels in ranking; not production integration."); + report["alignment_policy"] = json!(alignment::POLICY); + report["alignment_policy_sha256"] = json!(alignment::POLICY_SHA); + report["alignment_contract_sha256"] = json!(alignment::CONTRACT_SHA); + } report["phase"] = json!("document_encoding"); checkpoint(&output, &report)?; let start = Instant::now(); @@ -431,6 +464,39 @@ mod tests { .is_err()); } + #[test] + fn aligned_pool_preserves_all_competitors_and_marks_real_failures_ineligible() { + let hash = "a".repeat(64); + let assets: Vec<_> = ASSETS + .iter() + .map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.wav")})) + .collect(); + let raw = json!({"transcription":[{"text":" First. Second!","offsets":{"from":0,"to":1000},"tokens":[ + {"id":100,"text":" First.","offsets":{"from":100,"to":400}}, + {"id":101,"text":" Second!","offsets":{"from":600,"to":900}}]}]}); + let rows:Vec<_> = ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap(),"aligned_segments":alignment::reconstruct(&raw,1000,1000).unwrap()}]})).collect(); + let mut report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":"12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8","harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows,"alignment_policy":alignment::POLICY,"alignment_policy_sha256":alignment::POLICY_SHA,"alignment_contract_sha256":alignment::CONTRACT_SHA}); + let fixture = json!({"assets":assets}); + assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 10); + let mut forged = report.clone(); + forged["rows"][0]["windows"][0]["aligned_segments"][0]["start_ms"] = json!(1500); + assert!(eligible(&forged, &fixture).is_err()); + let window = &mut report["rows"][0]["windows"][0]; + window["actual_whisper_json"]["transcription"][0]["tokens"][0] + .as_object_mut() + .unwrap() + .remove("offsets"); + window["alignment_validation_error"] = json!("Missing token time"); + window.as_object_mut().unwrap().remove("aligned_segments"); + report["quality_failed"] = json!(true); + report["validation_failures"] = json!([{"error":"Missing token time"}]); + let (pool, excluded) = eligible(&report, &fixture).unwrap(); + assert_eq!(pool.len(), 8); + assert_eq!(excluded.len(), 1); + // Main derives quality_score_eligible solely from excluded.is_empty(). + assert!(!excluded.is_empty()); + } + #[test] fn source_identity_coverage_and_recognized_text_cannot_be_forged() { let hash = "a".repeat(64);