Measure complete CPU gated transcription and component costs

tools/speech-vad/test_compare.py

@@ -0,0 +1,44 @@
+import copy
+import unittest
+from compare import compare, ASSETS
+
+
+class Comparison(unittest.TestCase):
+    def reports(self):
+        common = {"complete": True, "quality_failed": False,
+                  **{key: "same" for key in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads")}}
+        window = {"segments": [{"text": "recognized"}], "decode_ms": 2, "asr_process_ms": 10}
+        baseline = {**common, "rows": [{"asset": asset, "source_sha256": asset, "windows": [copy.deepcopy(window)]} for asset in sorted(ASSETS)]}
+        gated = copy.deepcopy(baseline)
+        gated["comparison_eligible"] = True
+        gated["actual_vad_report"] = {"rows": [{"asset": asset, "decode_ms": 3, "process_ms": 5} for asset in sorted(ASSETS)]}
+        for row in gated["rows"]:
+            row["status"] = "no_speech"
+            row["windows"] = []
+        return baseline, gated
+
+    def test_no_speech_retains_competitors_and_vad_cost(self):
+        baseline, gated = self.reports()
+        result = compare(baseline, gated)
+        self.assertEqual(len(result["rows"]), 5)
+        self.assertEqual(result["rows"][0]["gated"]["asr_process_ms"], 0)
+        self.assertEqual(result["rows"][0]["vad_process_ms"], 5)
+
+    def test_ineligible_or_changed_pool_rejected(self):
+        for mutation in (lambda r: r.update(complete=False), lambda r: r.update(quality_failed=True),
+                         lambda r: r.update(comparison_eligible=False), lambda r: r.update(model_sha256="changed"),
+                         lambda r: r["rows"].pop()):
+            baseline, gated = self.reports()
+            mutation(gated)
+            with self.assertRaises(ValueError):
+                compare(baseline, gated)
+
+    def test_invalid_window_cannot_hide_behind_success(self):
+        baseline, gated = self.reports()
+        baseline["rows"][0]["windows"][0]["validation_error"] = "bad timestamp"
+        with self.assertRaises(ValueError):
+            compare(baseline, gated)
+
+
+if __name__ == "__main__":
+    unittest.main()