Measure complete CPU gated transcription and component costs

tools/speech-vad/compare.py

@@ -0,0 +1,47 @@
+"""Compare complete frozen CPU ASR pools; timing is not retrieval quality."""
+import argparse
+import json
+from pathlib import Path
+
+ASSETS = {"clip04", "music01", "music02", "sound01", "sound02"}
+
+
+def compare(baseline, gated):
+    for report in (baseline, gated):
+        if report.get("complete") is not True or report.get("quality_failed") is not False:
+            raise ValueError("Incomplete or structurally invalid comparison pool")
+        rows = report.get("rows", [])
+        if len(rows) != 5 or {row["asset"] for row in rows} != ASSETS:
+            raise ValueError("All five unique competitors required")
+    for field in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads"):
+        if baseline[field] != gated[field]:
+            raise ValueError("Comparison provenance differs: " + field)
+    if gated.get("comparison_eligible") is not True:
+        raise ValueError("Gated comparison is ineligible")
+    original = {row["asset"]: row for row in baseline["rows"]}
+    vad_rows = {row["asset"]: row for row in gated["actual_vad_report"]["rows"]}
+    results = []
+    for row in gated["rows"]:
+        old = original[row["asset"]]
+        if old["source_sha256"] != row["source_sha256"]:
+            raise ValueError("Changed source")
+        def stats(windows):
+            if any("validation_error" in window for window in windows):
+                raise ValueError("Invalid window must not be scored")
+            return {"windows": len(windows), "segments": sum(len(w["segments"]) for w in windows),
+                    "decode_ms": sum(w["decode_ms"] for w in windows),
+                    "asr_process_ms": sum(w["asr_process_ms"] for w in windows)}
+        results.append({"asset": row["asset"], "baseline": stats(old["windows"]),
+                        "gated": stats(row["windows"]), "status": row["status"],
+                        "vad_decode_ms": vad_rows[row["asset"]]["decode_ms"],
+                        "vad_process_ms": vad_rows[row["asset"]]["process_ms"]})
+    return {"scope": "Small sequential CPU diagnostic; process timings include model loading; not search latency or quality", "rows": results}
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("baseline", type=Path)
+    parser.add_argument("gated", type=Path)
+    args = parser.parse_args()
+    print(json.dumps(compare(json.loads(args.baseline.read_text(encoding="utf-8")),
+                             json.loads(args.gated.read_text(encoding="utf-8"))), indent=2))