Freeze independent DTW speech timing diagnostic and startup provenance

tools/speech-eval/README.md

@@ -89,3 +89,31 @@ Run only with the coordinator's available CPU inference slot. The previous
 base-only run took about 251 seconds including decode/process startup. Token
 alignment has not yet been timed; each ASR process retains the 600-second limit.
 No production behavior or quality claim follows from structural tests.
+
+## Separate DTW point-span comparison
+
+`dtw-policy.json` freezes a separate base experiment. Replace `--token-sentences`
+with `--dtw-sentences` and use a new output such as
+`.searchi/speech/base-dtw-sentences-v1.json`. This adds `--dtw base
+--no-flash-attn`: the pinned runtime disables DTW when flash attention is on.
+Native `t_dtw` is a centisecond point, approximately the moment a token is output,
+not a start/end interval. Multiply by ten; first and last ordinary-token points
+define an approximate sentence point span. One-token sentences can have zero span.
+Only source-window-bounded globally monotonic integer points are accepted.
+Original ordinary interval errors are retained separately and never repaired.
+Missing/invalid DTW points invalidate accepted scoring for the new comparison.
+Frozen ordinary-v1 output remains failed and unchanged.
+
+Pinned native default DTW graph workspace is 128 MiB; this is additional workspace,
+not a total memory ceiling. Attention storage and disabled flash attention can
+increase memory/time. Input remains 30 seconds, one CPU thread, 4 MiB output,
+4096 tokens/segment and 600 seconds/process; observed peak memory/runtime must
+be measured before production use. No inference has established this policy.
+Future harness reports capture the executing-source SHA at startup and preload
+alignment helpers before processing. The first ordinary-v1 run read its harness
+SHA at completion while the on-disk harness had changed; that field cannot prove
+the exact executing harness bytes. Its original raw token failures remain valid
+diagnostic observations, and its excluded window already prohibits an accepted
+quality score. Preserve that historical limitation instead of rewriting output.
+References: pinned [token data/API](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/include/whisper.h)
+and [DTW implementation](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/src/whisper.cpp).

tools/speech-eval/dtw-policy.json

@@ -0,0 +1,13 @@
+{
+  "schema": 1,
+  "policy": "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels",
+  "candidate": "base",
+  "worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8",
+  "fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
+  "assets": ["clip04", "music01", "music02", "sound01", "sound02"],
+  "query_count": 8,
+  "asr_flags": ["--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--output-json-full", "--dtw", "base", "--no-flash-attn"],
+  "point_units": "Native t_dtw centiseconds multiplied by ten; first/last ordinary-token points define approximate sentence point span, never native token duration",
+  "acceptance": "All sources/windows retained; ordinary interval failures retained separately; only valid source-window-bounded globally monotonic DTW points can produce accepted aligned candidates",
+  "production_default": false
+}

tools/speech-eval/dtw_alignment.py

@@ -0,0 +1,47 @@
+"""Frozen DTW sentence point spans, independent of ordinary token intervals."""
+from token_alignment import BOUNDARY, TEXT_TOKEN_LIMIT
+
+POLICY = "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels"
+
+
+def aligned_segments(document, offset_ms, duration_ms):
+    from run import segments
+    originals = segments(document, offset_ms, duration_ms)
+    raw_rows = [row for row in document["transcription"] if row["text"].strip()]
+    result = []
+    previous = 0
+    for index, (row, original) in enumerate(zip(raw_rows, originals)):
+        tokens = row.get("tokens")
+        if not isinstance(tokens, list) or not 0 < len(tokens) <= 4096:
+            raise ValueError("Missing or oversized DTW tokens")
+        accepted = []
+        for token in tokens:
+            token_id, text = token.get("id"), token.get("text")
+            if type(token_id) is not int or token_id < 0 or not isinstance(text, str):
+                raise ValueError("Invalid DTW token identity")
+            if token_id >= TEXT_TOKEN_LIMIT:
+                if not text.startswith("[_") or not text.endswith("]"):
+                    raise ValueError("Unrecognized special token")
+                continue
+            point = token.get("t_dtw")
+            if type(point) is not int or not 0 <= point * 10 <= duration_ms or point * 10 < previous:
+                raise ValueError("Missing, invalid or nonmonotonic DTW point")
+            previous = point * 10
+            accepted.append((text, previous))
+        if "".join(token[0] for token in accepted) != row["text"]:
+            raise ValueError("DTW text differs from original recognized segment")
+        phrase = []
+        for token_index, token in enumerate(accepted):
+            phrase.append(token)
+            if BOUNDARY.search(token[0]) or token_index == len(accepted) - 1:
+                text = "".join(part[0] for part in phrase)
+                if text.strip():
+                    result.append({"start_ms": offset_ms + phrase[0][1],
+                                   "end_ms": offset_ms + phrase[-1][1], "text": text,
+                                   "original_segment_index": index,
+                                   "original_start_ms": original["start_ms"],
+                                   "original_end_ms": original["end_ms"]})
+                phrase = []
+    if len(result) > 10000:
+        raise ValueError("Oversized DTW passages")
+    return result

tools/speech-eval/run.py

@@ -19,6 +19,8 @@ MODELS = {
 ASSETS = ("clip04", "music01", "music02", "sound01", "sound02")
 FIXTURE_SHA = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393"
 MAX_OUTPUT = 4 * 1024 * 1024
+# Capture before any expensive work; end-of-run reads can describe another file.
+HARNESS_SHA256_AT_START = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
 
 
 def sha256(path):
@@ -149,18 +151,32 @@ def main():
     parser.add_argument("--model", type=Path, required=True)
     parser.add_argument("--candidate", choices=MODELS, required=True)
     parser.add_argument("--token-sentences", action="store_true", help="Frozen ordinary token-time diagnostic; retains original segments")
+    parser.add_argument("--dtw-sentences", action="store_true", help="Separate frozen DTW point-span diagnostic")
     parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
     parser.add_argument("--scratch-parent", type=Path, required=True)
     parser.add_argument("--output", type=Path, required=True)
     args = parser.parse_args()
     if args.output.exists():
         raise ValueError("Never overwrite experiment evidence")
-    if args.token_sentences and args.candidate != "base":
+    if args.token_sentences and args.dtw_sentences:
+        raise ValueError("Select one separate alignment experiment")
+    alignment_mode = args.token_sentences or args.dtw_sentences
+    alignment_file = "dtw_alignment.py" if args.dtw_sentences else "token_alignment.py"
+    policy_file = "dtw-policy.json" if args.dtw_sentences else "token-policy.json"
+    if alignment_mode and args.candidate != "base":
         raise ValueError("Frozen token experiment accepts multilingual base only")
-    if args.token_sentences:
-        policy = json.loads(Path(__file__).with_name("token-policy.json").read_text(encoding="utf-8"))
+    if alignment_mode:
+        policy = json.loads(Path(__file__).with_name(policy_file).read_text(encoding="utf-8"))
         if policy["worker_sha256"] != args.whisper_sha256:
             raise ValueError("Frozen token experiment worker mismatch")
+        alignment_contract_sha = sha256(Path(__file__).with_name(alignment_file))
+        alignment_policy_sha = sha256(Path(__file__).with_name(policy_file))
+        # Load helpers now; future file edits cannot change a later lazy import.
+        if args.dtw_sentences:
+            from dtw_alignment import aligned_segments
+            from token_alignment import aligned_segments as ordinary_segments
+        else:
+            from token_alignment import aligned_segments
     size, digest = MODELS[args.candidate]
     if args.model.stat().st_size != size or sha256(args.model) != digest:
         raise ValueError("Model size/SHA verification failed")
@@ -209,31 +225,38 @@ def main():
                 else:
                     asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
                         "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
-                        "--output-json-full" if args.token_sentences else "--output-json",
+                        "--output-json-full" if alignment_mode else "--output-json",
+                        *(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []),
                         "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
                     raw = json.loads(output.read_text(encoding="utf-8"))
                     result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
                     result["windows"][-1]["decoded_frames"] = frames
-                    if args.token_sentences:
-                        from token_alignment import aligned_segments
+                    if alignment_mode:
+                        if args.dtw_sentences:
+                            try:
+                                ordinary_segments(raw, offset, length)
+                            except (ValueError, KeyError, TypeError) as error:
+                                result["windows"][-1]["ordinary_alignment_validation_error"] = str(error)
                         try:
                             result["windows"][-1]["aligned_segments"] = aligned_segments(raw, offset, length)
                         except (ValueError, KeyError, TypeError) as error:
                             result["windows"][-1]["alignment_validation_error"] = str(error)
                 args.output.parent.mkdir(parents=True, exist_ok=True)
                 checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
+                              "harness_sha256": HARNESS_SHA256_AT_START,
                               "model_sha256": digest, "worker_sha256": args.whisper_sha256,
                               "rows": [*rows, result]}
-                if args.token_sentences:
+                if alignment_mode:
                     checkpoint["alignment_policy"] = policy["policy"]
-                    checkpoint["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json"))
+                    checkpoint["alignment_policy_sha256"] = alignment_policy_sha
+                    checkpoint["alignment_contract_sha256"] = alignment_contract_sha
                 args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
             if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]:
                 raise ValueError("Source changed during recognition")
             rows.append(result)
     failures = [{"asset": row["asset"], "offset_ms": window["offset_ms"], "error": window["validation_error"]}
                 for row in rows for window in row["windows"] if "validation_error" in window]
-    if args.token_sentences:
+    if alignment_mode:
         failures.extend({"asset": row["asset"], "offset_ms": window["offset_ms"],
                          "error": window["alignment_validation_error"]}
                         for row in rows for window in row["windows"] if "alignment_validation_error" in window)
@@ -243,14 +266,14 @@ def main():
               "worker_sha256": args.whisper_sha256, "model_sha256": digest, "candidate": args.candidate,
               "ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
               "backend": "forced CPU", "threads": 1,
-              "harness_sha256": sha256(Path(__file__)),
+              "harness_sha256": HARNESS_SHA256_AT_START,
+              "harness_file_changed_during_run": sha256(Path(__file__)) != HARNESS_SHA256_AT_START,
               "policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
               "rows": rows}
-    if args.token_sentences:
-        from token_alignment import POLICY
-        report["alignment_policy"] = POLICY
-        report["alignment_contract_sha256"] = sha256(Path(__file__).with_name("token_alignment.py"))
-        report["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json"))
+    if alignment_mode:
+        report["alignment_policy"] = policy["policy"]
+        report["alignment_contract_sha256"] = alignment_contract_sha
+        report["alignment_policy_sha256"] = alignment_policy_sha
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
 

tools/speech-eval/test_contract.py

@@ -6,11 +6,24 @@ import tempfile
 import sys
 import os
 import wave
+import importlib.util
+import hashlib
 from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range
 from score import score, require_eligible
 
 
 class ContractTests(unittest.TestCase):
+    def test_harness_source_hash_is_captured_before_file_changes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "run-copy.py"
+            original = Path(__file__).with_name("run.py").read_bytes()
+            path.write_bytes(original)
+            spec = importlib.util.spec_from_file_location("speech_start_hash_test", path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            path.write_bytes(original + b"\n# changed after startup\n")
+            self.assertEqual(module.HARNESS_SHA256_AT_START, hashlib.sha256(original).hexdigest())
+            self.assertNotEqual(module.HARNESS_SHA256_AT_START, module.sha256(path))
     def test_scoring_rejects_incomplete_or_reduced_pool(self):
         for report in [{}, {"complete": False, "quality_score_eligible": True},
                        {"complete": True, "quality_score_eligible": False},

tools/speech-eval/test_dtw_alignment.py

@@ -0,0 +1,21 @@
+import unittest
+from dtw_alignment import aligned_segments
+
+
+class DtwTests(unittest.TestCase):
+    def raw(self):
+        return {"transcription": [{"text": " Hello. Again!", "offsets": {"from": 0, "to": 1000},
+            "tokens": [{"id": 100, "text": " Hello.", "t_dtw": 20, "offsets": {"from": 900, "to": 100}},
+                       {"id": 101, "text": " Again!", "t_dtw": 70}]}]}
+
+    def test_points_are_centiseconds_and_ordinary_reversal_not_repaired(self):
+        result = aligned_segments(self.raw(), 30000, 1000)
+        self.assertEqual([(r["start_ms"], r["end_ms"]) for r in result], [(30200, 30200), (30700, 30700)])
+        self.assertEqual("".join(r["text"] for r in result), " Hello. Again!")
+
+    def test_invalid_missing_fractional_and_nonmonotonic_points_rejected(self):
+        for point in [-1, 101, 1.5, None, 10]:
+            raw = self.raw()
+            raw["transcription"][0]["tokens"][1]["t_dtw"] = point
+            with self.assertRaises(ValueError):
+                aligned_segments(raw, 0, 1000)