import copy import unittest from compare import compare, ASSETS class Comparison(unittest.TestCase): def reports(self): common = {"complete": True, "quality_failed": False, **{key: "same" for key in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads")}} window = {"segments": [{"text": "recognized"}], "decode_ms": 2, "asr_process_ms": 10} baseline = {**common, "rows": [{"asset": asset, "source_sha256": asset, "windows": [copy.deepcopy(window)]} for asset in sorted(ASSETS)]} gated = copy.deepcopy(baseline) gated["comparison_eligible"] = True gated["actual_vad_report"] = {"rows": [{"asset": asset, "decode_ms": 3, "process_ms": 5} for asset in sorted(ASSETS)]} for row in gated["rows"]: row["status"] = "no_speech" row["windows"] = [] return baseline, gated def test_no_speech_retains_competitors_and_vad_cost(self): baseline, gated = self.reports() result = compare(baseline, gated) self.assertEqual(len(result["rows"]), 5) self.assertEqual(result["rows"][0]["gated"]["asr_process_ms"], 0) self.assertEqual(result["rows"][0]["vad_process_ms"], 5) def test_ineligible_or_changed_pool_rejected(self): for mutation in (lambda r: r.update(complete=False), lambda r: r.update(quality_failed=True), lambda r: r.update(comparison_eligible=False), lambda r: r.update(model_sha256="changed"), lambda r: r["rows"].pop()): baseline, gated = self.reports() mutation(gated) with self.assertRaises(ValueError): compare(baseline, gated) def test_invalid_window_cannot_hide_behind_success(self): baseline, gated = self.reports() baseline["rows"][0]["windows"][0]["validation_error"] = "bad timestamp" with self.assertRaises(ValueError): compare(baseline, gated) if __name__ == "__main__": unittest.main()