Freeze actual Google music metadata and precise speech timing diagnostics

tools/speech-eval/README.md

@@ -62,3 +62,30 @@ English recording do not validate German speech or general transcription quality
 
 Pinned CLI flag/output reference:
 [upstream cli.cpp](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/examples/cli/cli.cpp).
+
+## Frozen ordinary token-time comparison
+
+`token-policy.json` declares an internal base-only comparison before inference.
+Add `--token-sentences` to the existing command. The worker uses full JSON,
+ordinary token timestamps and unchanged recognition flags; DTW is disabled.
+The harness retains original raw JSON and segments, then adds exact recognized
+sentence substrings and their token intervals in `aligned_segments`. No label
+selects a sentence, source, word or start. All five sources and complete original
+window coverage remain mandatory; empty speech is valid.
+
+Structural tokens cannot become passage text. Ordinary text-token times must be
+present, ordered and inside the source segment/window; reconstructed token text
+must match original recognized bytes. Invalid alignment is retained as an error
+and invalidates an accepted score. No clipping, invented time or omitted failing
+competitor is permitted. Rankers must independently reconstruct these passages
+from raw JSON and verify policy/contract provenance before publishing quality.
+
+```powershell
+$env:SEARCHI_FORCE_CPU = '1'
+python tools/speech-eval/run.py --whisper .searchi/speech/build-cpu/bin/Release/whisper-cli.exe --whisper-sha256 12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8 --ffmpeg .searchi/media-runtime/bin/ffmpeg.exe --ffprobe .searchi/media-runtime/bin/ffprobe.exe --model .searchi/speech/models/ggml-base.bin --candidate base --token-sentences --scratch-parent .searchi/speech/token-scratch --output .searchi/speech/base-token-sentences-v1.json
+```
+
+Run only with the coordinator's available CPU inference slot. The previous
+base-only run took about 251 seconds including decode/process startup. Token
+alignment has not yet been timed; each ASR process retains the 600-second limit.
+No production behavior or quality claim follows from structural tests.

tools/speech-eval/run.py

@@ -148,10 +148,19 @@ def main():
     parser.add_argument("--ffprobe", type=Path, required=True)
     parser.add_argument("--model", type=Path, required=True)
     parser.add_argument("--candidate", choices=MODELS, required=True)
+    parser.add_argument("--token-sentences", action="store_true", help="Frozen ordinary token-time diagnostic; retains original segments")
     parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
     parser.add_argument("--scratch-parent", type=Path, required=True)
     parser.add_argument("--output", type=Path, required=True)
     args = parser.parse_args()
+    if args.output.exists():
+        raise ValueError("Never overwrite experiment evidence")
+    if args.token_sentences and args.candidate != "base":
+        raise ValueError("Frozen token experiment accepts multilingual base only")
+    if args.token_sentences:
+        policy = json.loads(Path(__file__).with_name("token-policy.json").read_text(encoding="utf-8"))
+        if policy["worker_sha256"] != args.whisper_sha256:
+            raise ValueError("Frozen token experiment worker mismatch")
     size, digest = MODELS[args.candidate]
     if args.model.stat().st_size != size or sha256(args.model) != digest:
         raise ValueError("Model size/SHA verification failed")
@@ -200,20 +209,34 @@ def main():
                 else:
                     asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
                         "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
-                        "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
+                        "--output-json-full" if args.token_sentences else "--output-json",
+                        "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
                     raw = json.loads(output.read_text(encoding="utf-8"))
                     result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
                     result["windows"][-1]["decoded_frames"] = frames
+                    if args.token_sentences:
+                        from token_alignment import aligned_segments
+                        try:
+                            result["windows"][-1]["aligned_segments"] = aligned_segments(raw, offset, length)
+                        except (ValueError, KeyError, TypeError) as error:
+                            result["windows"][-1]["alignment_validation_error"] = str(error)
                 args.output.parent.mkdir(parents=True, exist_ok=True)
                 checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
                               "model_sha256": digest, "worker_sha256": args.whisper_sha256,
                               "rows": [*rows, result]}
+                if args.token_sentences:
+                    checkpoint["alignment_policy"] = policy["policy"]
+                    checkpoint["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json"))
                 args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
             if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]:
                 raise ValueError("Source changed during recognition")
             rows.append(result)
     failures = [{"asset": row["asset"], "offset_ms": window["offset_ms"], "error": window["validation_error"]}
                 for row in rows for window in row["windows"] if "validation_error" in window]
+    if args.token_sentences:
+        failures.extend({"asset": row["asset"], "offset_ms": window["offset_ms"],
+                         "error": window["alignment_validation_error"]}
+                        for row in rows for window in row["windows"] if "alignment_validation_error" in window)
     report = {"version": 1, "complete": True, "quality_failed": bool(failures), "validation_failures": failures,
               "scope": "Actual ASR diagnostic, not Google retrieval or production integration",
               "fixture_sha256": sha256(args.fixture), "upstream_commit": UPSTREAM,
@@ -223,6 +246,11 @@ def main():
               "harness_sha256": sha256(Path(__file__)),
               "policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
               "rows": rows}
+    if args.token_sentences:
+        from token_alignment import POLICY
+        report["alignment_policy"] = POLICY
+        report["alignment_contract_sha256"] = sha256(Path(__file__).with_name("token_alignment.py"))
+        report["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json"))
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
 

tools/speech-eval/test_token_alignment.py

@@ -0,0 +1,60 @@
+"""Structural tests only; actual word-time quality requires genuine inference."""
+import unittest
+from token_alignment import aligned_segments
+
+
+def fixture():
+    tokens = [{"id": 100 + index, "text": text,
+               "offsets": {"from": start, "to": end}}
+              for index, (text, start, end) in enumerate([
+                  (" Hello", 100, 400), (" world.", 400, 900),
+                  (" Guten", 1100, 1400), (" Tag!", 1400, 1900)])]
+    return {"transcription": [{"text": " Hello world. Guten Tag!",
+                               "offsets": {"from": 0, "to": 2000}, "tokens": tokens}]}
+
+
+class TokenAlignmentTests(unittest.TestCase):
+    def test_exact_sentences_absolute_nonzero_source_coordinates(self):
+        result = aligned_segments(fixture(), 30000, 2000)
+        self.assertEqual([row["text"] for row in result], [" Hello world.", " Guten Tag!"])
+        self.assertEqual([(row["start_ms"], row["end_ms"]) for row in result],
+                         [(30100, 30900), (31100, 31900)])
+        self.assertEqual(result[0]["original_start_ms"], 30000)
+
+    def test_structural_tokens_never_enter_candidates(self):
+        raw = fixture()
+        raw["transcription"][0]["tokens"].insert(0, {"id": 50364, "text": "[_BEG_]"})
+        self.assertEqual(len(aligned_segments(raw, 0, 2000)), 2)
+
+    def test_missing_nonmonotonic_outside_segment_and_invalid_special_rejected(self):
+        mutations = [lambda token: token.pop("offsets"),
+                     lambda token: token.update(offsets={"from": -1, "to": 100}),
+                     lambda token: token.update(offsets={"from": 300, "to": 100}),
+                     lambda token: token.update(offsets={"from": 0, "to": 2500}),
+                     lambda token: token.update(id=50364, text="ordinary words"),
+                     lambda token: token.update(text=" changed")]
+        for mutate in mutations:
+            raw = fixture()
+            mutate(raw["transcription"][0]["tokens"][1])
+            with self.assertRaises(ValueError):
+                aligned_segments(raw, 0, 2000)
+
+    def test_no_punctuation_preserves_whole_text(self):
+        raw = fixture()
+        for token in raw["transcription"][0]["tokens"]:
+            token["text"] = token["text"].rstrip(".!")
+        raw["transcription"][0]["text"] = "".join(t["text"] for t in raw["transcription"][0]["tokens"])
+        result = aligned_segments(raw, 0, 2000)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["text"], raw["transcription"][0]["text"])
+
+    def test_empty_speech_is_valid_and_missing_tokens_are_rejected(self):
+        self.assertEqual(aligned_segments({"transcription": []}, 0, 1000), [])
+        raw = fixture()
+        del raw["transcription"][0]["tokens"]
+        with self.assertRaises(ValueError):
+            aligned_segments(raw, 0, 2000)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/speech-eval/token-policy.json

@@ -0,0 +1,14 @@
+{
+  "schema": 1,
+  "policy": "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels",
+  "candidate": "base",
+  "worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8",
+  "fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
+  "assets": ["clip04", "music01", "music02", "sound01", "sound02"],
+  "query_count": 8,
+  "asr_flags": ["--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--output-json-full"],
+  "boundaries": "Sentence-final punctuation in ordinary token text; remainder retained, exact recognized text unchanged",
+  "acceptance": "Every original source/window retained; invalid token time or text invalidates accepted score; no label-driven passage selection",
+  "comparison": "Frozen base segment baseline; file top1, strict timestamp-start hits, interval overlap, recognition omissions and processing cost reported separately",
+  "production_default": false
+}

tools/speech-eval/token_alignment.py

@@ -0,0 +1,54 @@
+"""Strict ordinary Whisper token-time sentence subdivision, never label driven."""
+import re
+
+POLICY = "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels"
+# Pinned multilingual Whisper vocabulary: ordinary text ids precede EOT.
+TEXT_TOKEN_LIMIT = 50257
+BOUNDARY = re.compile(r'[.!?。！？]["\u201d\u2019\')\]]*$')
+
+
+def aligned_segments(document, offset_ms, duration_ms):
+    from run import segments
+    originals = segments(document, offset_ms, duration_ms)
+    result = []
+    raw_rows = [row for row in document["transcription"] if row["text"].strip()]
+    for index, (row, original) in enumerate(zip(raw_rows, originals)):
+        tokens = row.get("tokens")
+        if not isinstance(tokens, list) or not 0 < len(tokens) <= 4096:
+            raise ValueError("Missing or oversized token times")
+        accepted = []
+        previous = 0
+        for token in tokens:
+            token_id, text = token.get("id"), token.get("text")
+            if type(token_id) is not int or token_id < 0 or not isinstance(text, str):
+                raise ValueError("Invalid token identity")
+            # Structural tokens must never become recognized text candidates.
+            if token_id >= TEXT_TOKEN_LIMIT:
+                if not text.startswith("[_") or not text.endswith("]"):
+                    raise ValueError("Unrecognized special token")
+                continue
+            times = token.get("offsets", {})
+            start, end = times.get("from"), times.get("to")
+            if (type(start) is not int or type(end) is not int
+                    or not 0 <= start <= end <= duration_ms or start < previous
+                    or not row["offsets"]["from"] <= start <= end <= row["offsets"]["to"]):
+                raise ValueError("Invalid ordinary token timestamps")
+            previous = start
+            accepted.append((text, start, end))
+        if "".join(token[0] for token in accepted) != row["text"]:
+            raise ValueError("Token text differs from original recognized segment")
+        phrase = []
+        for token_index, token in enumerate(accepted):
+            phrase.append(token)
+            if BOUNDARY.search(token[0]) or token_index == len(accepted) - 1:
+                text = "".join(part[0] for part in phrase)
+                if text.strip():
+                    result.append({"start_ms": offset_ms + phrase[0][1],
+                                   "end_ms": offset_ms + phrase[-1][2], "text": text,
+                                   "original_segment_index": index,
+                                   "original_start_ms": original["start_ms"],
+                                   "original_end_ms": original["end_ms"]})
+                phrase = []
+    if len(result) > 10000:
+        raise ValueError("Oversized aligned passage pool")
+    return result