44 lines
2.0 KiB
Python
44 lines
2.0 KiB
Python
|
import copy
|
||
|
|
import unittest
|
||
|
|
from compare import compare, ASSETS
|
||
|
|
|
||
|
|
|
||
|
|
class Comparison(unittest.TestCase):
|
||
|
|
def reports(self):
|
||
|
|
common = {"complete": True, "quality_failed": False,
|
||
|
|
**{key: "same" for key in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads")}}
|
||
|
|
window = {"segments": [{"text": "recognized"}], "decode_ms": 2, "asr_process_ms": 10}
|
||
|
|
baseline = {**common, "rows": [{"asset": asset, "source_sha256": asset, "windows": [copy.deepcopy(window)]} for asset in sorted(ASSETS)]}
|
||
|
|
gated = copy.deepcopy(baseline)
|
||
|
|
gated["comparison_eligible"] = True
|
||
|
|
gated["actual_vad_report"] = {"rows": [{"asset": asset, "decode_ms": 3, "process_ms": 5} for asset in sorted(ASSETS)]}
|
||
|
|
for row in gated["rows"]:
|
||
|
|
row["status"] = "no_speech"
|
||
|
|
row["windows"] = []
|
||
|
|
return baseline, gated
|
||
|
|
|
||
|
|
def test_no_speech_retains_competitors_and_vad_cost(self):
|
||
|
|
baseline, gated = self.reports()
|
||
|
|
result = compare(baseline, gated)
|
||
|
|
self.assertEqual(len(result["rows"]), 5)
|
||
|
|
self.assertEqual(result["rows"][0]["gated"]["asr_process_ms"], 0)
|
||
|
|
self.assertEqual(result["rows"][0]["vad_process_ms"], 5)
|
||
|
|
|
||
|
|
def test_ineligible_or_changed_pool_rejected(self):
|
||
|
|
for mutation in (lambda r: r.update(complete=False), lambda r: r.update(quality_failed=True),
|
||
|
|
lambda r: r.update(comparison_eligible=False), lambda r: r.update(model_sha256="changed"),
|
||
|
|
lambda r: r["rows"].pop()):
|
||
|
|
baseline, gated = self.reports()
|
||
|
|
mutation(gated)
|
||
|
|
with self.assertRaises(ValueError):
|
||
|
|
compare(baseline, gated)
|
||
|
|
|
||
|
|
def test_invalid_window_cannot_hide_behind_success(self):
|
||
|
|
baseline, gated = self.reports()
|
||
|
|
baseline["rows"][0]["windows"][0]["validation_error"] = "bad timestamp"
|
||
|
|
with self.assertRaises(ValueError):
|
||
|
|
compare(baseline, gated)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|