diff --git a/tools/speech-eval/README.md b/tools/speech-eval/README.md index 886ed89..20b2d4f 100644 --- a/tools/speech-eval/README.md +++ b/tools/speech-eval/README.md @@ -115,5 +115,15 @@ SHA at completion while the on-disk harness had changed; that field cannot prove the exact executing harness bytes. Its original raw token failures remain valid diagnostic observations, and its excluded window already prohibits an accepted quality score. Preserve that historical limitation instead of rewriting output. +The first DTW run stopped on strict UTF-8 parsing in the second piano window +(offset 30000 ms): native JSON bytes at 1137–1138 were invalid. Its temporary +raw output was not retained by that harness, so exact malformed bytes/native +cause cannot be established. The partial checkpoint retains all three NASA +windows (21 accepted DTW sentence spans, including one independently recorded +ordinary-interval failure) and the first piano window. Missing remaining windows +and sources prohibit any accepted retrieval score. It was not rerun or decoded +lossily. Subsequent harnesses retain bounded exact failing output bytes under +the owned scratch parent with SHA/size/parse position, and atomically record a +failed incomplete checkpoint before terminating; they do not skip competitors. References: pinned [token data/API](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/include/whisper.h) and [DTW implementation](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/src/whisper.cpp). diff --git a/tools/speech-eval/run.py b/tools/speech-eval/run.py index 7447ab6..e1a608a 100644 --- a/tools/speech-eval/run.py +++ b/tools/speech-eval/run.py @@ -31,6 +31,50 @@ def sha256(path): return digest.hexdigest() +class AsrOutputError(ValueError): + """Bounded raw output retained; callers must keep coverage incomplete.""" + def __init__(self, failure): + super().__init__("ASR output failed strict UTF-8/JSON parsing; raw evidence retained") + self.failure = failure + + +def checkpoint_report(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, + prefix="speech-checkpoint-", suffix=".tmp", delete=False) as temporary: + temporary_path = Path(temporary.name) + json.dump(document, temporary, indent=2, ensure_ascii=False) + temporary.write("\n") + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, path) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def parse_asr_output(path, retained_parent): + with path.open("rb") as stream: + raw = stream.read(MAX_OUTPUT + 1) + if len(raw) > MAX_OUTPUT: + raise ValueError("ASR output exceeds parse budget") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + with tempfile.NamedTemporaryFile(prefix="speech-asr-output-failure-", suffix=".bin", + dir=retained_parent, delete=False) as retained: + retained.write(raw) + failure = {"error_type": type(error).__name__, "raw_output_path": retained.name, + "raw_output_bytes": len(raw), "raw_output_sha256": hashlib.sha256(raw).hexdigest()} + if isinstance(error, UnicodeDecodeError): + failure.update(byte_start=error.start, byte_end=error.end, reason=error.reason) + else: + failure.update(character_position=error.pos, line=error.lineno, column=error.colno) + raise AsrOutputError(failure) from error + + def stop(child): if os.name == "nt": subprocess.run(["taskkill", "/PID", str(child.pid), "/T", "/F"], @@ -228,7 +272,25 @@ def main(): "--output-json-full" if alignment_mode else "--output-json", *(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []), "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600) - raw = json.loads(output.read_text(encoding="utf-8")) + try: + raw = parse_asr_output(output, args.scratch_parent) + except AsrOutputError as error: + result["windows"].append({"offset_ms": offset, "duration_ms": length, + "decode_ms": decode_ms, "asr_process_ms": asr_ms, + "decoded_frames": frames, "segments": [], + "error_stage": "asr_output_parse", "validation_error": str(error), + "actual_output_failure": error.failure}) + failed = {"complete": False, "quality_failed": True, + "comparison_eligible": False, "fixture_sha256": sha256(args.fixture), + "model_sha256": digest, "worker_sha256": args.whisper_sha256, + "harness_sha256": HARNESS_SHA256_AT_START, "rows": [*rows, result], + "failure": {"asset": asset["id"], "offset_ms": offset, **error.failure}} + if alignment_mode: + failed.update(alignment_policy=policy["policy"], + alignment_policy_sha256=alignment_policy_sha, + alignment_contract_sha256=alignment_contract_sha) + checkpoint_report(args.output, failed) + raise result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms)) result["windows"][-1]["decoded_frames"] = frames if alignment_mode: @@ -250,7 +312,7 @@ def main(): checkpoint["alignment_policy"] = policy["policy"] checkpoint["alignment_policy_sha256"] = alignment_policy_sha checkpoint["alignment_contract_sha256"] = alignment_contract_sha - args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + checkpoint_report(args.output, checkpoint) if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]: raise ValueError("Source changed during recognition") rows.append(result) @@ -275,7 +337,7 @@ def main(): report["alignment_contract_sha256"] = alignment_contract_sha report["alignment_policy_sha256"] = alignment_policy_sha args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + checkpoint_report(args.output, report) if __name__ == "__main__": diff --git a/tools/speech-eval/test_contract.py b/tools/speech-eval/test_contract.py index e5ab1b3..9809fb1 100644 --- a/tools/speech-eval/test_contract.py +++ b/tools/speech-eval/test_contract.py @@ -10,9 +10,33 @@ import importlib.util import hashlib from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range from score import score, require_eligible +from run import parse_asr_output, AsrOutputError, MAX_OUTPUT, checkpoint_report class ContractTests(unittest.TestCase): + def test_atomic_checkpoint_failure_preserves_previous_evidence(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "report.json" + checkpoint_report(path, {"complete": False, "rows": []}) + old = path.read_bytes() + with self.assertRaises(TypeError): + checkpoint_report(path, {"invalid": object()}) + self.assertEqual(path.read_bytes(), old) + def test_invalid_asr_utf8_and_json_retain_exact_bounded_bytes(self): + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) + path = parent / "output.json" + for raw in [b'{"text":"\xe2\x28\xa1"}', b'{"text":']: + path.write_bytes(raw) + with self.assertRaises(AsrOutputError) as caught: + parse_asr_output(path, parent) + failure = caught.exception.failure + self.assertEqual(Path(failure["raw_output_path"]).read_bytes(), raw) + self.assertEqual(failure["raw_output_sha256"], hashlib.sha256(raw).hexdigest()) + self.assertEqual(failure["raw_output_bytes"], len(raw)) + path.write_bytes(b"x" * (MAX_OUTPUT + 1)) + with self.assertRaises(ValueError): + parse_asr_output(path, parent) def test_harness_source_hash_is_captured_before_file_changes(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "run-copy.py"