Retain bounded invalid ASR output and atomic failure evidence

tools/speech-eval/README.md

@@ -115,5 +115,15 @@ SHA at completion while the on-disk harness had changed; that field cannot prove
 the exact executing harness bytes. Its original raw token failures remain valid
 diagnostic observations, and its excluded window already prohibits an accepted
 quality score. Preserve that historical limitation instead of rewriting output.
+The first DTW run stopped on strict UTF-8 parsing in the second piano window
+(offset 30000 ms): native JSON bytes at 1137–1138 were invalid. Its temporary
+raw output was not retained by that harness, so exact malformed bytes/native
+cause cannot be established. The partial checkpoint retains all three NASA
+windows (21 accepted DTW sentence spans, including one independently recorded
+ordinary-interval failure) and the first piano window. Missing remaining windows
+and sources prohibit any accepted retrieval score. It was not rerun or decoded
+lossily. Subsequent harnesses retain bounded exact failing output bytes under
+the owned scratch parent with SHA/size/parse position, and atomically record a
+failed incomplete checkpoint before terminating; they do not skip competitors.
 References: pinned [token data/API](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/include/whisper.h)
 and [DTW implementation](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/src/whisper.cpp).

tools/speech-eval/run.py

@@ -31,6 +31,50 @@ def sha256(path):
     return digest.hexdigest()
 
 
+class AsrOutputError(ValueError):
+    """Bounded raw output retained; callers must keep coverage incomplete."""
+    def __init__(self, failure):
+        super().__init__("ASR output failed strict UTF-8/JSON parsing; raw evidence retained")
+        self.failure = failure
+
+
+def checkpoint_report(path, document):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary_path = None
+    try:
+        with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
+                                         prefix="speech-checkpoint-", suffix=".tmp", delete=False) as temporary:
+            temporary_path = Path(temporary.name)
+            json.dump(document, temporary, indent=2, ensure_ascii=False)
+            temporary.write("\n")
+            temporary.flush()
+            os.fsync(temporary.fileno())
+        os.replace(temporary_path, path)
+    finally:
+        if temporary_path is not None:
+            temporary_path.unlink(missing_ok=True)
+
+
+def parse_asr_output(path, retained_parent):
+    with path.open("rb") as stream:
+        raw = stream.read(MAX_OUTPUT + 1)
+    if len(raw) > MAX_OUTPUT:
+        raise ValueError("ASR output exceeds parse budget")
+    try:
+        return json.loads(raw.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        with tempfile.NamedTemporaryFile(prefix="speech-asr-output-failure-", suffix=".bin",
+                                         dir=retained_parent, delete=False) as retained:
+            retained.write(raw)
+        failure = {"error_type": type(error).__name__, "raw_output_path": retained.name,
+                   "raw_output_bytes": len(raw), "raw_output_sha256": hashlib.sha256(raw).hexdigest()}
+        if isinstance(error, UnicodeDecodeError):
+            failure.update(byte_start=error.start, byte_end=error.end, reason=error.reason)
+        else:
+            failure.update(character_position=error.pos, line=error.lineno, column=error.colno)
+        raise AsrOutputError(failure) from error
+
+
 def stop(child):
     if os.name == "nt":
         subprocess.run(["taskkill", "/PID", str(child.pid), "/T", "/F"],
@@ -228,7 +272,25 @@ def main():
                         "--output-json-full" if alignment_mode else "--output-json",
                         *(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []),
                         "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
-                    raw = json.loads(output.read_text(encoding="utf-8"))
+                    try:
+                        raw = parse_asr_output(output, args.scratch_parent)
+                    except AsrOutputError as error:
+                        result["windows"].append({"offset_ms": offset, "duration_ms": length,
+                            "decode_ms": decode_ms, "asr_process_ms": asr_ms,
+                            "decoded_frames": frames, "segments": [],
+                            "error_stage": "asr_output_parse", "validation_error": str(error),
+                            "actual_output_failure": error.failure})
+                        failed = {"complete": False, "quality_failed": True,
+                            "comparison_eligible": False, "fixture_sha256": sha256(args.fixture),
+                            "model_sha256": digest, "worker_sha256": args.whisper_sha256,
+                            "harness_sha256": HARNESS_SHA256_AT_START, "rows": [*rows, result],
+                            "failure": {"asset": asset["id"], "offset_ms": offset, **error.failure}}
+                        if alignment_mode:
+                            failed.update(alignment_policy=policy["policy"],
+                                alignment_policy_sha256=alignment_policy_sha,
+                                alignment_contract_sha256=alignment_contract_sha)
+                        checkpoint_report(args.output, failed)
+                        raise
                     result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
                     result["windows"][-1]["decoded_frames"] = frames
                     if alignment_mode:
@@ -250,7 +312,7 @@ def main():
                     checkpoint["alignment_policy"] = policy["policy"]
                     checkpoint["alignment_policy_sha256"] = alignment_policy_sha
                     checkpoint["alignment_contract_sha256"] = alignment_contract_sha
-                args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+                checkpoint_report(args.output, checkpoint)
             if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]:
                 raise ValueError("Source changed during recognition")
             rows.append(result)
@@ -275,7 +337,7 @@ def main():
         report["alignment_contract_sha256"] = alignment_contract_sha
         report["alignment_policy_sha256"] = alignment_policy_sha
     args.output.parent.mkdir(parents=True, exist_ok=True)
-    args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    checkpoint_report(args.output, report)
 
 
 if __name__ == "__main__":

tools/speech-eval/test_contract.py

@@ -10,9 +10,33 @@ import importlib.util
 import hashlib
 from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range
 from score import score, require_eligible
+from run import parse_asr_output, AsrOutputError, MAX_OUTPUT, checkpoint_report
 
 
 class ContractTests(unittest.TestCase):
+    def test_atomic_checkpoint_failure_preserves_previous_evidence(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "report.json"
+            checkpoint_report(path, {"complete": False, "rows": []})
+            old = path.read_bytes()
+            with self.assertRaises(TypeError):
+                checkpoint_report(path, {"invalid": object()})
+            self.assertEqual(path.read_bytes(), old)
+    def test_invalid_asr_utf8_and_json_retain_exact_bounded_bytes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            parent = Path(directory)
+            path = parent / "output.json"
+            for raw in [b'{"text":"\xe2\x28\xa1"}', b'{"text":']:
+                path.write_bytes(raw)
+                with self.assertRaises(AsrOutputError) as caught:
+                    parse_asr_output(path, parent)
+                failure = caught.exception.failure
+                self.assertEqual(Path(failure["raw_output_path"]).read_bytes(), raw)
+                self.assertEqual(failure["raw_output_sha256"], hashlib.sha256(raw).hexdigest())
+                self.assertEqual(failure["raw_output_bytes"], len(raw))
+            path.write_bytes(b"x" * (MAX_OUTPUT + 1))
+            with self.assertRaises(ValueError):
+                parse_asr_output(path, parent)
     def test_harness_source_hash_is_captured_before_file_changes(self):
         with tempfile.TemporaryDirectory() as directory:
             path = Path(directory) / "run-copy.py"