Freeze independent DTW speech timing diagnostic and startup provenance

tools/speech-rank/README.md

@@ -75,3 +75,11 @@ use the existing executable command with input
 `.searchi/speech/base-token-sentences-v1.json` and a new output filename, then the
 same frozen eight-query scorer. Do not overwrite old results or select passages
 with relevance labels.
+
+The separate `dtw-token-sentences-v1` experiment independently validates native
+DTW centisecond points and uses the first/last token points as approximate
+sentence spans. These are not token durations. Ordinary interval failures remain
+in the embedded ASR evidence and are independently checked; they do not authorize
+fabricating or clamping DTW points. Missing/invalid DTW points exclude a window
+and prevent accepted scoring. Use a separate DTW report/output and retain the
+failed ordinary-v1 run. The frozen five sources/eight queries remain unchanged.

tools/speech-rank/src/alignment.rs

@@ -7,6 +7,11 @@ pub const POLICY: &str =
     "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels";
 pub const POLICY_SHA: &str = "5bc603e792a3abcc2370d4d1f7a13f538f808fb3b959b506a16d5823d3d7a126";
 pub const CONTRACT_SHA: &str = "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520";
+pub const DTW_POLICY: &str =
+    "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels";
+pub const DTW_POLICY_SHA: &str = "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5";
+pub const DTW_CONTRACT_SHA: &str =
+    "61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f";
 
 pub fn enabled(asr: &Value) -> Result<bool> {
     if asr.get("alignment_policy").is_none() {
@@ -18,9 +23,12 @@ pub fn enabled(asr: &Value) -> Result<bool> {
         return Ok(false);
     }
     ensure!(
-        asr["alignment_policy"] == POLICY
+        ((asr["alignment_policy"] == POLICY
             && asr["alignment_policy_sha256"] == POLICY_SHA
-            && asr["alignment_contract_sha256"] == CONTRACT_SHA
+            && asr["alignment_contract_sha256"] == CONTRACT_SHA)
+            || (asr["alignment_policy"] == DTW_POLICY
+                && asr["alignment_policy_sha256"] == DTW_POLICY_SHA
+                && asr["alignment_contract_sha256"] == DTW_CONTRACT_SHA))
             && asr["candidate"] == "base"
             && asr["worker_sha256"]
                 == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8"
@@ -38,12 +46,17 @@ fn boundary(text: &str) -> bool {
 }
 
 pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
+    reconstruct_mode(raw, offset, duration, false)
+}
+
+pub fn reconstruct_mode(raw: &Value, offset: u64, duration: u64, dtw: bool) -> Result<Vec<Value>> {
     let originals = parsed_segments(raw, offset, duration)?;
     let rows = raw["transcription"]
         .as_array()
         .context("Missing transcription")?;
     let mut result = Vec::new();
     let mut original_index = 0;
+    let mut dtw_previous = 0;
     for row in rows {
         let text = row["text"].as_str().context("Missing original text")?;
         if text.trim().is_empty() {
@@ -70,14 +83,29 @@ pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>
                 );
                 continue;
             }
-            let start = number(&token["offsets"]["from"])?;
-            let end = number(&token["offsets"]["to"])?;
+            let (start, end) = if dtw {
+                let point = number(&token["t_dtw"])?
+                    .checked_mul(10)
+                    .context("DTW point overflow")?;
+                ensure!(
+                    point >= dtw_previous && point <= duration,
+                    "Invalid DTW source point"
+                );
+                dtw_previous = point;
+                (point, point)
+            } else {
+                (
+                    number(&token["offsets"]["from"])?,
+                    number(&token["offsets"]["to"])?,
+                )
+            };
             ensure!(
                 start >= previous
                     && start <= end
                     && end <= duration
-                    && start >= number(&row["offsets"]["from"])?
-                    && end <= number(&row["offsets"]["to"])?,
+                    && (dtw
+                        || (start >= number(&row["offsets"]["from"])?
+                            && end <= number(&row["offsets"]["to"])?)),
                 "Invalid ordinary token time"
             );
             previous = start;
@@ -187,4 +215,23 @@ mod tests {
         changed["candidate"] = json!("small");
         assert!(enabled(&changed).is_err());
     }
+    #[test]
+    fn dtw_points_are_independent_bounded_monotonic_centiseconds() {
+        let mut raw = raw();
+        for (index, point) in [(1, 20), (2, 70), (3, 90)] {
+            raw["transcription"][0]["tokens"][index]["t_dtw"] = json!(point);
+        }
+        raw["transcription"][0]["tokens"][1]["offsets"] = json!({"from":900,"to":100});
+        assert!(reconstruct(&raw, 0, 2000).is_err());
+        let result = reconstruct_mode(&raw, 30000, 2000, true).unwrap();
+        assert_eq!(result[0]["start_ms"], 30200);
+        assert_eq!(result[0]["end_ms"], 30200);
+        assert_eq!(result[1]["start_ms"], 30700);
+        assert_eq!(result[1]["end_ms"], 30900);
+        for point in [json!(-1), json!(201), json!(1.5), json!(10), Value::Null] {
+            let mut changed = raw.clone();
+            changed["transcription"][0]["tokens"][2]["t_dtw"] = point;
+            assert!(reconstruct_mode(&changed, 0, 2000, true).is_err());
+        }
+    }
 }

tools/speech-rank/src/main.rs

@@ -190,8 +190,31 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
                 "Normalized ASR segments differ from actual output"
             );
             let actual = if token_alignment {
-                let rebuilt =
-                    alignment::reconstruct(&window["actual_whisper_json"], offset, duration);
+                let dtw = asr["alignment_policy"] == alignment::DTW_POLICY;
+                if dtw {
+                    let ordinary_failed =
+                        alignment::reconstruct(&window["actual_whisper_json"], offset, duration)
+                            .is_err();
+                    ensure!(
+                        window.get("ordinary_alignment_validation_error").is_some()
+                            == ordinary_failed,
+                        "Ordinary token failure evidence inconsistent"
+                    );
+                    if ordinary_failed {
+                        ensure!(
+                            window["ordinary_alignment_validation_error"]
+                                .as_str()
+                                .is_some_and(|error| !error.is_empty()),
+                            "Invalid ordinary failure reason"
+                        );
+                    }
+                }
+                let rebuilt = alignment::reconstruct_mode(
+                    &window["actual_whisper_json"],
+                    offset,
+                    duration,
+                    dtw,
+                );
                 if let Some(error) = window.get("alignment_validation_error") {
                     ensure!(
                         error.as_str().is_some_and(|e| !e.is_empty()) && rebuilt.is_err(),
@@ -347,10 +370,10 @@ fn main() -> Result<()> {
     report["quality_score_eligible"] = json!(excluded.is_empty());
     report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
     if alignment::enabled(&asr)? {
-        report["scope"] = json!("Diagnostic ordinary token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate token intervals, no labels in ranking; not production integration.");
-        report["alignment_policy"] = json!(alignment::POLICY);
-        report["alignment_policy_sha256"] = json!(alignment::POLICY_SHA);
-        report["alignment_contract_sha256"] = json!(alignment::CONTRACT_SHA);
+        report["scope"] = json!("Diagnostic token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate ordinary intervals or DTW point spans as declared in alignment_policy, no labels in ranking; not production integration.");
+        report["alignment_policy"] = asr["alignment_policy"].clone();
+        report["alignment_policy_sha256"] = asr["alignment_policy_sha256"].clone();
+        report["alignment_contract_sha256"] = asr["alignment_contract_sha256"].clone();
     }
     report["phase"] = json!("document_encoding");
     checkpoint(&output, &report)?;