Freeze actual Google music metadata and precise speech timing diagnostics

tools/speech-eval/test_token_alignment.py

@@ -0,0 +1,60 @@
+"""Structural tests only; actual word-time quality requires genuine inference."""
+import unittest
+from token_alignment import aligned_segments
+
+
+def fixture():
+    tokens = [{"id": 100 + index, "text": text,
+               "offsets": {"from": start, "to": end}}
+              for index, (text, start, end) in enumerate([
+                  (" Hello", 100, 400), (" world.", 400, 900),
+                  (" Guten", 1100, 1400), (" Tag!", 1400, 1900)])]
+    return {"transcription": [{"text": " Hello world. Guten Tag!",
+                               "offsets": {"from": 0, "to": 2000}, "tokens": tokens}]}
+
+
+class TokenAlignmentTests(unittest.TestCase):
+    def test_exact_sentences_absolute_nonzero_source_coordinates(self):
+        result = aligned_segments(fixture(), 30000, 2000)
+        self.assertEqual([row["text"] for row in result], [" Hello world.", " Guten Tag!"])
+        self.assertEqual([(row["start_ms"], row["end_ms"]) for row in result],
+                         [(30100, 30900), (31100, 31900)])
+        self.assertEqual(result[0]["original_start_ms"], 30000)
+
+    def test_structural_tokens_never_enter_candidates(self):
+        raw = fixture()
+        raw["transcription"][0]["tokens"].insert(0, {"id": 50364, "text": "[_BEG_]"})
+        self.assertEqual(len(aligned_segments(raw, 0, 2000)), 2)
+
+    def test_missing_nonmonotonic_outside_segment_and_invalid_special_rejected(self):
+        mutations = [lambda token: token.pop("offsets"),
+                     lambda token: token.update(offsets={"from": -1, "to": 100}),
+                     lambda token: token.update(offsets={"from": 300, "to": 100}),
+                     lambda token: token.update(offsets={"from": 0, "to": 2500}),
+                     lambda token: token.update(id=50364, text="ordinary words"),
+                     lambda token: token.update(text=" changed")]
+        for mutate in mutations:
+            raw = fixture()
+            mutate(raw["transcription"][0]["tokens"][1])
+            with self.assertRaises(ValueError):
+                aligned_segments(raw, 0, 2000)
+
+    def test_no_punctuation_preserves_whole_text(self):
+        raw = fixture()
+        for token in raw["transcription"][0]["tokens"]:
+            token["text"] = token["text"].rstrip(".!")
+        raw["transcription"][0]["text"] = "".join(t["text"] for t in raw["transcription"][0]["tokens"])
+        result = aligned_segments(raw, 0, 2000)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["text"], raw["transcription"][0]["text"])
+
+    def test_empty_speech_is_valid_and_missing_tokens_are_rejected(self):
+        self.assertEqual(aligned_segments({"transcription": []}, 0, 1000), [])
+        raw = fixture()
+        del raw["transcription"][0]["tokens"]
+        with self.assertRaises(ValueError):
+            aligned_segments(raw, 0, 2000)
+
+
+if __name__ == "__main__":
+    unittest.main()