Freeze independent DTW speech timing diagnostic and startup provenance

tools/speech-eval/README.md

@@ -89,3 +89,31 @@ Run only with the coordinator's available CPU inference slot. The previous
 base-only run took about 251 seconds including decode/process startup. Token
 alignment has not yet been timed; each ASR process retains the 600-second limit.
 No production behavior or quality claim follows from structural tests.
+
+## Separate DTW point-span comparison
+
+`dtw-policy.json` freezes a separate base experiment. Replace `--token-sentences`
+with `--dtw-sentences` and use a new output such as
+`.searchi/speech/base-dtw-sentences-v1.json`. This adds `--dtw base
+--no-flash-attn`: the pinned runtime disables DTW when flash attention is on.
+Native `t_dtw` is a centisecond point, approximately the moment a token is output,
+not a start/end interval. Multiply by ten; first and last ordinary-token points
+define an approximate sentence point span. One-token sentences can have zero span.
+Only source-window-bounded globally monotonic integer points are accepted.
+Original ordinary interval errors are retained separately and never repaired.
+Missing/invalid DTW points invalidate accepted scoring for the new comparison.
+Frozen ordinary-v1 output remains failed and unchanged.
+
+Pinned native default DTW graph workspace is 128 MiB; this is additional workspace,
+not a total memory ceiling. Attention storage and disabled flash attention can
+increase memory/time. Input remains 30 seconds, one CPU thread, 4 MiB output,
+4096 tokens/segment and 600 seconds/process; observed peak memory/runtime must
+be measured before production use. No inference has established this policy.
+Future harness reports capture the executing-source SHA at startup and preload
+alignment helpers before processing. The first ordinary-v1 run read its harness
+SHA at completion while the on-disk harness had changed; that field cannot prove
+the exact executing harness bytes. Its original raw token failures remain valid
+diagnostic observations, and its excluded window already prohibits an accepted
+quality score. Preserve that historical limitation instead of rewriting output.
+References: pinned [token data/API](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/include/whisper.h)
+and [DTW implementation](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/src/whisper.cpp).

tools/speech-eval/dtw-policy.json

@@ -0,0 +1,13 @@
+{
+  "schema": 1,
+  "policy": "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels",
+  "candidate": "base",
+  "worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8",
+  "fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
+  "assets": ["clip04", "music01", "music02", "sound01", "sound02"],
+  "query_count": 8,
+  "asr_flags": ["--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--output-json-full", "--dtw", "base", "--no-flash-attn"],
+  "point_units": "Native t_dtw centiseconds multiplied by ten; first/last ordinary-token points define approximate sentence point span, never native token duration",
+  "acceptance": "All sources/windows retained; ordinary interval failures retained separately; only valid source-window-bounded globally monotonic DTW points can produce accepted aligned candidates",
+  "production_default": false
+}

tools/speech-eval/dtw_alignment.py

@@ -0,0 +1,47 @@
+"""Frozen DTW sentence point spans, independent of ordinary token intervals."""
+from token_alignment import BOUNDARY, TEXT_TOKEN_LIMIT
+
+POLICY = "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels"
+
+
+def aligned_segments(document, offset_ms, duration_ms):
+    from run import segments
+    originals = segments(document, offset_ms, duration_ms)
+    raw_rows = [row for row in document["transcription"] if row["text"].strip()]
+    result = []
+    previous = 0
+    for index, (row, original) in enumerate(zip(raw_rows, originals)):
+        tokens = row.get("tokens")
+        if not isinstance(tokens, list) or not 0 < len(tokens) <= 4096:
+            raise ValueError("Missing or oversized DTW tokens")
+        accepted = []
+        for token in tokens:
+            token_id, text = token.get("id"), token.get("text")
+            if type(token_id) is not int or token_id < 0 or not isinstance(text, str):
+                raise ValueError("Invalid DTW token identity")
+            if token_id >= TEXT_TOKEN_LIMIT:
+                if not text.startswith("[_") or not text.endswith("]"):
+                    raise ValueError("Unrecognized special token")
+                continue
+            point = token.get("t_dtw")
+            if type(point) is not int or not 0 <= point * 10 <= duration_ms or point * 10 < previous:
+                raise ValueError("Missing, invalid or nonmonotonic DTW point")
+            previous = point * 10
+            accepted.append((text, previous))
+        if "".join(token[0] for token in accepted) != row["text"]:
+            raise ValueError("DTW text differs from original recognized segment")
+        phrase = []
+        for token_index, token in enumerate(accepted):
+            phrase.append(token)
+            if BOUNDARY.search(token[0]) or token_index == len(accepted) - 1:
+                text = "".join(part[0] for part in phrase)
+                if text.strip():
+                    result.append({"start_ms": offset_ms + phrase[0][1],
+                                   "end_ms": offset_ms + phrase[-1][1], "text": text,
+                                   "original_segment_index": index,
+                                   "original_start_ms": original["start_ms"],
+                                   "original_end_ms": original["end_ms"]})
+                phrase = []
+    if len(result) > 10000:
+        raise ValueError("Oversized DTW passages")
+    return result

tools/speech-eval/run.py

@@ -19,6 +19,8 @@ MODELS = {
 ASSETS = ("clip04", "music01", "music02", "sound01", "sound02")
 FIXTURE_SHA = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393"
 MAX_OUTPUT = 4 * 1024 * 1024
+# Capture before any expensive work; end-of-run reads can describe another file.
+HARNESS_SHA256_AT_START = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
 
 
 def sha256(path):
@@ -149,18 +151,32 @@ def main():
     parser.add_argument("--model", type=Path, required=True)
     parser.add_argument("--candidate", choices=MODELS, required=True)
     parser.add_argument("--token-sentences", action="store_true", help="Frozen ordinary token-time diagnostic; retains original segments")
+    parser.add_argument("--dtw-sentences", action="store_true", help="Separate frozen DTW point-span diagnostic")
     parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
     parser.add_argument("--scratch-parent", type=Path, required=True)
     parser.add_argument("--output", type=Path, required=True)
     args = parser.parse_args()
     if args.output.exists():
         raise ValueError("Never overwrite experiment evidence")
-    if args.token_sentences and args.candidate != "base":
+    if args.token_sentences and args.dtw_sentences:
+        raise ValueError("Select one separate alignment experiment")
+    alignment_mode = args.token_sentences or args.dtw_sentences
+    alignment_file = "dtw_alignment.py" if args.dtw_sentences else "token_alignment.py"
+    policy_file = "dtw-policy.json" if args.dtw_sentences else "token-policy.json"
+    if alignment_mode and args.candidate != "base":
         raise ValueError("Frozen token experiment accepts multilingual base only")
-    if args.token_sentences:
-        policy = json.loads(Path(__file__).with_name("token-policy.json").read_text(encoding="utf-8"))
+    if alignment_mode:
+        policy = json.loads(Path(__file__).with_name(policy_file).read_text(encoding="utf-8"))
         if policy["worker_sha256"] != args.whisper_sha256:
             raise ValueError("Frozen token experiment worker mismatch")
+        alignment_contract_sha = sha256(Path(__file__).with_name(alignment_file))
+        alignment_policy_sha = sha256(Path(__file__).with_name(policy_file))
+        # Load helpers now; future file edits cannot change a later lazy import.
+        if args.dtw_sentences:
+            from dtw_alignment import aligned_segments
+            from token_alignment import aligned_segments as ordinary_segments
+        else:
+            from token_alignment import aligned_segments
     size, digest = MODELS[args.candidate]
     if args.model.stat().st_size != size or sha256(args.model) != digest:
         raise ValueError("Model size/SHA verification failed")
@@ -209,31 +225,38 @@ def main():
                 else:
                     asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
                         "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
-                        "--output-json-full" if args.token_sentences else "--output-json",
+                        "--output-json-full" if alignment_mode else "--output-json",
+                        *(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []),
                         "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
                     raw = json.loads(output.read_text(encoding="utf-8"))
                     result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
                     result["windows"][-1]["decoded_frames"] = frames
-                    if args.token_sentences:
-                        from token_alignment import aligned_segments
+                    if alignment_mode:
+                        if args.dtw_sentences:
+                            try:
+                                ordinary_segments(raw, offset, length)
+                            except (ValueError, KeyError, TypeError) as error:
+                                result["windows"][-1]["ordinary_alignment_validation_error"] = str(error)
                         try:
                             result["windows"][-1]["aligned_segments"] = aligned_segments(raw, offset, length)
                         except (ValueError, KeyError, TypeError) as error:
                             result["windows"][-1]["alignment_validation_error"] = str(error)
                 args.output.parent.mkdir(parents=True, exist_ok=True)
                 checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
+                              "harness_sha256": HARNESS_SHA256_AT_START,
                               "model_sha256": digest, "worker_sha256": args.whisper_sha256,
                               "rows": [*rows, result]}
-                if args.token_sentences:
+                if alignment_mode:
                     checkpoint["alignment_policy"] = policy["policy"]
-                    checkpoint["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json"))
+                    checkpoint["alignment_policy_sha256"] = alignment_policy_sha
+                    checkpoint["alignment_contract_sha256"] = alignment_contract_sha
                 args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
             if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]:
                 raise ValueError("Source changed during recognition")
             rows.append(result)
     failures = [{"asset": row["asset"], "offset_ms": window["offset_ms"], "error": window["validation_error"]}
                 for row in rows for window in row["windows"] if "validation_error" in window]
-    if args.token_sentences:
+    if alignment_mode:
         failures.extend({"asset": row["asset"], "offset_ms": window["offset_ms"],
                          "error": window["alignment_validation_error"]}
                         for row in rows for window in row["windows"] if "alignment_validation_error" in window)
@@ -243,14 +266,14 @@ def main():
               "worker_sha256": args.whisper_sha256, "model_sha256": digest, "candidate": args.candidate,
               "ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
               "backend": "forced CPU", "threads": 1,
-              "harness_sha256": sha256(Path(__file__)),
+              "harness_sha256": HARNESS_SHA256_AT_START,
+              "harness_file_changed_during_run": sha256(Path(__file__)) != HARNESS_SHA256_AT_START,
               "policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
               "rows": rows}
-    if args.token_sentences:
-        from token_alignment import POLICY
-        report["alignment_policy"] = POLICY
-        report["alignment_contract_sha256"] = sha256(Path(__file__).with_name("token_alignment.py"))
-        report["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json"))
+    if alignment_mode:
+        report["alignment_policy"] = policy["policy"]
+        report["alignment_contract_sha256"] = alignment_contract_sha
+        report["alignment_policy_sha256"] = alignment_policy_sha
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
 

tools/speech-eval/test_contract.py

@@ -6,11 +6,24 @@ import tempfile
 import sys
 import os
 import wave
+import importlib.util
+import hashlib
 from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range
 from score import score, require_eligible
 
 
 class ContractTests(unittest.TestCase):
+    def test_harness_source_hash_is_captured_before_file_changes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "run-copy.py"
+            original = Path(__file__).with_name("run.py").read_bytes()
+            path.write_bytes(original)
+            spec = importlib.util.spec_from_file_location("speech_start_hash_test", path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            path.write_bytes(original + b"\n# changed after startup\n")
+            self.assertEqual(module.HARNESS_SHA256_AT_START, hashlib.sha256(original).hexdigest())
+            self.assertNotEqual(module.HARNESS_SHA256_AT_START, module.sha256(path))
     def test_scoring_rejects_incomplete_or_reduced_pool(self):
         for report in [{}, {"complete": False, "quality_score_eligible": True},
                        {"complete": True, "quality_score_eligible": False},

tools/speech-eval/test_dtw_alignment.py

@@ -0,0 +1,21 @@
+import unittest
+from dtw_alignment import aligned_segments
+
+
+class DtwTests(unittest.TestCase):
+    def raw(self):
+        return {"transcription": [{"text": " Hello. Again!", "offsets": {"from": 0, "to": 1000},
+            "tokens": [{"id": 100, "text": " Hello.", "t_dtw": 20, "offsets": {"from": 900, "to": 100}},
+                       {"id": 101, "text": " Again!", "t_dtw": 70}]}]}
+
+    def test_points_are_centiseconds_and_ordinary_reversal_not_repaired(self):
+        result = aligned_segments(self.raw(), 30000, 1000)
+        self.assertEqual([(r["start_ms"], r["end_ms"]) for r in result], [(30200, 30200), (30700, 30700)])
+        self.assertEqual("".join(r["text"] for r in result), " Hello. Again!")
+
+    def test_invalid_missing_fractional_and_nonmonotonic_points_rejected(self):
+        for point in [-1, 101, 1.5, None, 10]:
+            raw = self.raw()
+            raw["transcription"][0]["tokens"][1]["t_dtw"] = point
+            with self.assertRaises(ValueError):
+                aligned_segments(raw, 0, 1000)

tools/speech-rank/README.md

@@ -75,3 +75,11 @@ use the existing executable command with input
 `.searchi/speech/base-token-sentences-v1.json` and a new output filename, then the
 same frozen eight-query scorer. Do not overwrite old results or select passages
 with relevance labels.
+
+The separate `dtw-token-sentences-v1` experiment independently validates native
+DTW centisecond points and uses the first/last token points as approximate
+sentence spans. These are not token durations. Ordinary interval failures remain
+in the embedded ASR evidence and are independently checked; they do not authorize
+fabricating or clamping DTW points. Missing/invalid DTW points exclude a window
+and prevent accepted scoring. Use a separate DTW report/output and retain the
+failed ordinary-v1 run. The frozen five sources/eight queries remain unchanged.

tools/speech-rank/src/alignment.rs

@@ -7,6 +7,11 @@ pub const POLICY: &str =
     "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels";
 pub const POLICY_SHA: &str = "5bc603e792a3abcc2370d4d1f7a13f538f808fb3b959b506a16d5823d3d7a126";
 pub const CONTRACT_SHA: &str = "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520";
+pub const DTW_POLICY: &str =
+    "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels";
+pub const DTW_POLICY_SHA: &str = "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5";
+pub const DTW_CONTRACT_SHA: &str =
+    "61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f";
 
 pub fn enabled(asr: &Value) -> Result<bool> {
     if asr.get("alignment_policy").is_none() {
@@ -18,9 +23,12 @@ pub fn enabled(asr: &Value) -> Result<bool> {
         return Ok(false);
     }
     ensure!(
-        asr["alignment_policy"] == POLICY
+        ((asr["alignment_policy"] == POLICY
             && asr["alignment_policy_sha256"] == POLICY_SHA
-            && asr["alignment_contract_sha256"] == CONTRACT_SHA
+            && asr["alignment_contract_sha256"] == CONTRACT_SHA)
+            || (asr["alignment_policy"] == DTW_POLICY
+                && asr["alignment_policy_sha256"] == DTW_POLICY_SHA
+                && asr["alignment_contract_sha256"] == DTW_CONTRACT_SHA))
             && asr["candidate"] == "base"
             && asr["worker_sha256"]
                 == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8"
@@ -38,12 +46,17 @@ fn boundary(text: &str) -> bool {
 }
 
 pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
+    reconstruct_mode(raw, offset, duration, false)
+}
+
+pub fn reconstruct_mode(raw: &Value, offset: u64, duration: u64, dtw: bool) -> Result<Vec<Value>> {
     let originals = parsed_segments(raw, offset, duration)?;
     let rows = raw["transcription"]
         .as_array()
         .context("Missing transcription")?;
     let mut result = Vec::new();
     let mut original_index = 0;
+    let mut dtw_previous = 0;
     for row in rows {
         let text = row["text"].as_str().context("Missing original text")?;
         if text.trim().is_empty() {
@@ -70,14 +83,29 @@ pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>
                 );
                 continue;
             }
-            let start = number(&token["offsets"]["from"])?;
-            let end = number(&token["offsets"]["to"])?;
+            let (start, end) = if dtw {
+                let point = number(&token["t_dtw"])?
+                    .checked_mul(10)
+                    .context("DTW point overflow")?;
+                ensure!(
+                    point >= dtw_previous && point <= duration,
+                    "Invalid DTW source point"
+                );
+                dtw_previous = point;
+                (point, point)
+            } else {
+                (
+                    number(&token["offsets"]["from"])?,
+                    number(&token["offsets"]["to"])?,
+                )
+            };
             ensure!(
                 start >= previous
                     && start <= end
                     && end <= duration
-                    && start >= number(&row["offsets"]["from"])?
-                    && end <= number(&row["offsets"]["to"])?,
+                    && (dtw
+                        || (start >= number(&row["offsets"]["from"])?
+                            && end <= number(&row["offsets"]["to"])?)),
                 "Invalid ordinary token time"
             );
             previous = start;
@@ -187,4 +215,23 @@ mod tests {
         changed["candidate"] = json!("small");
         assert!(enabled(&changed).is_err());
     }
+    #[test]
+    fn dtw_points_are_independent_bounded_monotonic_centiseconds() {
+        let mut raw = raw();
+        for (index, point) in [(1, 20), (2, 70), (3, 90)] {
+            raw["transcription"][0]["tokens"][index]["t_dtw"] = json!(point);
+        }
+        raw["transcription"][0]["tokens"][1]["offsets"] = json!({"from":900,"to":100});
+        assert!(reconstruct(&raw, 0, 2000).is_err());
+        let result = reconstruct_mode(&raw, 30000, 2000, true).unwrap();
+        assert_eq!(result[0]["start_ms"], 30200);
+        assert_eq!(result[0]["end_ms"], 30200);
+        assert_eq!(result[1]["start_ms"], 30700);
+        assert_eq!(result[1]["end_ms"], 30900);
+        for point in [json!(-1), json!(201), json!(1.5), json!(10), Value::Null] {
+            let mut changed = raw.clone();
+            changed["transcription"][0]["tokens"][2]["t_dtw"] = point;
+            assert!(reconstruct_mode(&changed, 0, 2000, true).is_err());
+        }
+    }
 }

tools/speech-rank/src/main.rs

@@ -190,8 +190,31 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
                 "Normalized ASR segments differ from actual output"
             );
             let actual = if token_alignment {
-                let rebuilt =
-                    alignment::reconstruct(&window["actual_whisper_json"], offset, duration);
+                let dtw = asr["alignment_policy"] == alignment::DTW_POLICY;
+                if dtw {
+                    let ordinary_failed =
+                        alignment::reconstruct(&window["actual_whisper_json"], offset, duration)
+                            .is_err();
+                    ensure!(
+                        window.get("ordinary_alignment_validation_error").is_some()
+                            == ordinary_failed,
+                        "Ordinary token failure evidence inconsistent"
+                    );
+                    if ordinary_failed {
+                        ensure!(
+                            window["ordinary_alignment_validation_error"]
+                                .as_str()
+                                .is_some_and(|error| !error.is_empty()),
+                            "Invalid ordinary failure reason"
+                        );
+                    }
+                }
+                let rebuilt = alignment::reconstruct_mode(
+                    &window["actual_whisper_json"],
+                    offset,
+                    duration,
+                    dtw,
+                );
                 if let Some(error) = window.get("alignment_validation_error") {
                     ensure!(
                         error.as_str().is_some_and(|e| !e.is_empty()) && rebuilt.is_err(),
@@ -347,10 +370,10 @@ fn main() -> Result<()> {
     report["quality_score_eligible"] = json!(excluded.is_empty());
     report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
     if alignment::enabled(&asr)? {
-        report["scope"] = json!("Diagnostic ordinary token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate token intervals, no labels in ranking; not production integration.");
-        report["alignment_policy"] = json!(alignment::POLICY);
-        report["alignment_policy_sha256"] = json!(alignment::POLICY_SHA);
-        report["alignment_contract_sha256"] = json!(alignment::CONTRACT_SHA);
+        report["scope"] = json!("Diagnostic token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate ordinary intervals or DTW point spans as declared in alignment_policy, no labels in ranking; not production integration.");
+        report["alignment_policy"] = asr["alignment_policy"].clone();
+        report["alignment_policy_sha256"] = asr["alignment_policy_sha256"].clone();
+        report["alignment_contract_sha256"] = asr["alignment_contract_sha256"].clone();
     }
     report["phase"] = json!("document_encoding");
     checkpoint(&output, &report)?;