Measure real CPU VAD on complete frozen media pool

tools/speech-vad/run.py

@@ -0,0 +1,85 @@
+"""Frozen complete media-pool CPU VAD experiment; no ASR or ranking changes."""
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+
+spec = importlib.util.spec_from_file_location("speech_contract", Path(__file__).parents[1] / "speech-eval/run.py")
+speech = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(speech)
+MODEL_SHA = "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987"
+
+
+def source_intersection(interval, samples, origin_ms):
+    start, end = interval
+    if start < 0 or end <= start or int(start) != start or int(end) != end:
+        raise ValueError("Invalid raw centisecond interval")
+    raw_start, raw_end = int(start) * 160, int(end) * 160
+    bounded_start, bounded_end = min(raw_start, samples), min(raw_end, samples)
+    return {"raw_cs": interval, "source_samples": [bounded_start, bounded_end],
+            "source_absolute_ms": [origin_ms + bounded_start / 16, origin_ms + bounded_end / 16],
+            "tail_padding_samples": max(0, raw_end - samples)}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    for option in ("worker", "model", "ffmpeg", "ffprobe", "output", "scratch-parent"):
+        parser.add_argument("--" + option, type=Path, required=True)
+    parser.add_argument("--worker-sha256", required=True)
+    parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
+    args = parser.parse_args()
+    if args.output.exists():
+        raise ValueError("Never overwrite experiment evidence")
+    if speech.sha256(args.worker) != args.worker_sha256 or speech.sha256(args.model) != MODEL_SHA:
+        raise ValueError("Frozen worker/model mismatch")
+    if speech.sha256(args.fixture) != speech.FIXTURE_SHA:
+        raise ValueError("Frozen media fixture mismatch")
+    fixture = json.loads(args.fixture.read_text(encoding="utf-8"))
+    selected = [asset for asset in fixture["assets"] if asset["id"] in speech.ASSETS]
+    if sorted(asset["id"] for asset in selected) != sorted(speech.ASSETS):
+        raise ValueError("Incomplete or duplicate competitor pool")
+    rows = []
+    args.scratch_parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="vad-proof-", dir=args.scratch_parent) as scratch_dir:
+        scratch = Path(scratch_dir)
+        for asset in selected:
+            source = args.fixture.parent / asset["path"]
+            if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]:
+                raise ValueError("Frozen source mismatch")
+            probe_ms = speech.run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
+                "-show_entries", "stream=index,codec_type,duration,start_time", "-of", "json", str(source.resolve())], scratch, "probe", [], 30)
+            probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
+            start, end, index = speech.primary_audio_range(probe)
+            pcm = scratch / "audio.f32"
+            command = speech.decode_command(args.ffmpeg, source, start, end - start, pcm, index)
+            command[command.index("pcm_s16le")] = "pcm_f32le"
+            command[-1:-1] = ["-f", "f32le"]
+            decode_ms = speech.run_owned(command, scratch, "decode", [(pcm, 16000 * 600 * 4)], 60)
+            samples = pcm.stat().st_size // 4
+            if pcm.stat().st_size % 4 or abs(samples - (end - start) * 16) > 16:
+                raise ValueError("Incomplete decoded coverage")
+            process_ms = speech.run_owned([str(args.worker.resolve()), str(args.model.resolve()), str(pcm)], scratch, "vad", [], 120)
+            log = (scratch / "vad.log").read_text(encoding="utf-8")
+            report = json.loads(log.strip().splitlines()[-1])
+            if report["samples"] != samples or report["threads"] != 1 or report["gpu"] is not False:
+                raise ValueError("Unexpected native contract")
+            if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]:
+                raise ValueError("Source changed during detection")
+            rows.append({"asset": asset["id"], "source_sha256": asset["sha256"], "probe": probe,
+                         "probe_ms": probe_ms, "decode_ms": decode_ms, "process_ms": process_ms,
+                         "native": report, "native_log": log,
+                         "intervals": [source_intersection(interval, samples, start) for interval in report["intervals_cs"]]})
+    result = {"schema": 1, "complete": True, "scope": "VAD only; no ASR quality or retrieval claim",
+              "fixture_sha256": speech.FIXTURE_SHA, "worker_sha256": args.worker_sha256,
+              "model_sha256": MODEL_SHA, "harness_sha256": speech.sha256(Path(__file__)),
+              "speech_contract_sha256": speech.sha256(Path(spec.origin)),
+              "ffmpeg_sha256": speech.sha256(args.ffmpeg), "ffprobe_sha256": speech.sha256(args.ffprobe), "rows": rows}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("x", encoding="utf-8") as stream:
+        json.dump(result, stream, indent=2)
+        stream.write("\n")
+
+
+if __name__ == "__main__":
+    main()