Measure complete CPU gated transcription and component costs

tools/speech-vad/README.md

@@ -30,3 +30,9 @@ minimum speech 250 ms, silence 100 ms, maximum speech FLT_MAX, padding 30 ms,
 overlap 0.1 seconds. It does not establish model accuracy or a production
 speech gate. Compare the complete frozen speech/music/sound pool before judging
 quality; preserve original Google audio retrieval regardless of transcription.
+
+`gated_asr.py` runs the frozen base transcription experiment from the verified
+VAD report. It preserves all five competitors, no-speech status, original source
+intervals and raw ASR logs. `compare.py BASELINE GATED` reports component timing
+and segment counts only; it rejects incomplete or invalid pools and does not
+score retrieval quality. See the protocol document for actual evidence.

tools/speech-vad/compare.py

@@ -0,0 +1,47 @@
+"""Compare complete frozen CPU ASR pools; timing is not retrieval quality."""
+import argparse
+import json
+from pathlib import Path
+
+ASSETS = {"clip04", "music01", "music02", "sound01", "sound02"}
+
+
+def compare(baseline, gated):
+    for report in (baseline, gated):
+        if report.get("complete") is not True or report.get("quality_failed") is not False:
+            raise ValueError("Incomplete or structurally invalid comparison pool")
+        rows = report.get("rows", [])
+        if len(rows) != 5 or {row["asset"] for row in rows} != ASSETS:
+            raise ValueError("All five unique competitors required")
+    for field in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads"):
+        if baseline[field] != gated[field]:
+            raise ValueError("Comparison provenance differs: " + field)
+    if gated.get("comparison_eligible") is not True:
+        raise ValueError("Gated comparison is ineligible")
+    original = {row["asset"]: row for row in baseline["rows"]}
+    vad_rows = {row["asset"]: row for row in gated["actual_vad_report"]["rows"]}
+    results = []
+    for row in gated["rows"]:
+        old = original[row["asset"]]
+        if old["source_sha256"] != row["source_sha256"]:
+            raise ValueError("Changed source")
+        def stats(windows):
+            if any("validation_error" in window for window in windows):
+                raise ValueError("Invalid window must not be scored")
+            return {"windows": len(windows), "segments": sum(len(w["segments"]) for w in windows),
+                    "decode_ms": sum(w["decode_ms"] for w in windows),
+                    "asr_process_ms": sum(w["asr_process_ms"] for w in windows)}
+        results.append({"asset": row["asset"], "baseline": stats(old["windows"]),
+                        "gated": stats(row["windows"]), "status": row["status"],
+                        "vad_decode_ms": vad_rows[row["asset"]]["decode_ms"],
+                        "vad_process_ms": vad_rows[row["asset"]]["process_ms"]})
+    return {"scope": "Small sequential CPU diagnostic; process timings include model loading; not search latency or quality", "rows": results}
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("baseline", type=Path)
+    parser.add_argument("gated", type=Path)
+    args = parser.parse_args()
+    print(json.dumps(compare(json.loads(args.baseline.read_text(encoding="utf-8")),
+                             json.loads(args.gated.read_text(encoding="utf-8"))), indent=2))

tools/speech-vad/gated_asr.py

@@ -0,0 +1,170 @@
+"""Frozen VAD-gated CPU ASR diagnostic; never changes product search."""
+import argparse
+import importlib.util
+import json
+import math
+from pathlib import Path
+import tempfile
+
+spec = importlib.util.spec_from_file_location("gated_speech_contract", Path(__file__).parents[1] / "speech-eval/run.py")
+speech = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(speech)
+vad_spec = importlib.util.spec_from_file_location("gated_vad_contract", Path(__file__).with_name("run.py"))
+vad = importlib.util.module_from_spec(vad_spec)
+vad_spec.loader.exec_module(vad)
+VAD_SHA = "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df"
+VAD_WORKER_SHA = "28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19"
+WHISPER_SHA = "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8"
+POLICY = "gated-vad-v1;source-intersections;integer-ms-tail-remainder-recorded;contiguous30s;independent-process;auto-language;segment-times"
+
+
+def interval_windows(intervals, samples, origin):
+    """Never concatenate intervals or reapply padding; retain sub-ms tail loss."""
+    result = []
+    previous = 0
+    for index, interval in enumerate(intervals):
+        expected = vad.source_intersection(interval["raw_cs"], samples, origin)
+        if interval != expected:
+            raise ValueError("VAD source intersection differs from raw evidence")
+        start, end = interval["source_samples"]
+        if start < previous:
+            raise ValueError("Overlapping or unordered source intersections")
+        previous = end
+        # Raw starts are integer centiseconds; only the final bounded tail can
+        # have a sample fraction that whisper's integer-ms serializer cannot hold.
+        first, last = origin + start // 16, origin + end // 16
+        for offset in range(first, last, 30000):
+            result.append({"offset_ms": offset, "duration_ms": min(30000, last - offset),
+                           "vad_interval_index": index, "vad_source_intersection": interval,
+                           "unrepresented_tail_samples": end % 16})
+    return result
+
+
+def validated_rows(document, fixture):
+    if (document.get("schema") != 1 or document.get("complete") is not True
+            or document.get("fixture_sha256") != speech.FIXTURE_SHA
+            or document.get("model_sha256") != vad.MODEL_SHA
+            or document.get("worker_sha256") != VAD_WORKER_SHA):
+        raise ValueError("Unrecognized frozen VAD provenance")
+    rows = document.get("rows", [])
+    if sorted(row["asset"] for row in rows) != sorted(speech.ASSETS):
+        raise ValueError("Incomplete or duplicate VAD competitor pool")
+    assets = {asset["id"]: asset for asset in fixture["assets"]}
+    for row in rows:
+        if row["source_sha256"] != assets[row["asset"]]["sha256"]:
+            raise ValueError("VAD source hash mismatch")
+        native = row["native"]
+        start, end, _ = speech.primary_audio_range(row["probe"])
+        samples = native["samples"]
+        probabilities = native["probabilities"]
+        if (type(samples) is not int or samples <= 0 or abs(samples - (end-start)*16) > 16
+                or native["sample_rate"] != 16000 or native["threads"] != 1 or native["gpu"] is not False
+                or len(probabilities) != (samples + 511) // 512
+                or any(type(p) not in (int, float) or not math.isfinite(p) or not 0 <= p <= 1 for p in probabilities)
+                or [interval["raw_cs"] for interval in row["intervals"]] != native["intervals_cs"]):
+            raise ValueError("Invalid VAD native coverage or CPU contract")
+        interval_windows(row["intervals"], samples, start)
+    return rows
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    for option in ("whisper", "model", "ffmpeg", "ffprobe", "vad-report", "scratch-parent", "output"):
+        parser.add_argument("--" + option, type=Path, required=True)
+    parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
+    args = parser.parse_args()
+    if args.output.exists():
+        raise ValueError("Never overwrite experiment evidence")
+    if speech.sha256(args.vad_report) != VAD_SHA or speech.sha256(args.fixture) != speech.FIXTURE_SHA:
+        raise ValueError("Frozen VAD/fixture SHA mismatch")
+    size, model_sha = speech.MODELS["base"]
+    if args.model.stat().st_size != size or speech.sha256(args.model) != model_sha or speech.sha256(args.whisper) != WHISPER_SHA:
+        raise ValueError("Frozen ASR worker/model mismatch")
+    if args.vad_report.stat().st_size > 16 * 1024 * 1024:
+        raise ValueError("Oversized VAD evidence")
+    vad_bytes = args.vad_report.read_bytes()
+    document = json.loads(vad_bytes)
+    for name in ("ffmpeg", "ffprobe"):
+        if speech.sha256(getattr(args, name)) != document[name + "_sha256"]:
+            raise ValueError("Decoder provenance differs from frozen VAD")
+    fixture = json.loads(args.fixture.read_text(encoding="utf-8"))
+    vad_rows = validated_rows(document, fixture)
+    assets = {asset["id"]: asset for asset in fixture["assets"]}
+    report = {"version": 1, "complete": False, "quality_failed": False, "validation_failures": [],
+              "comparison_eligible": False, "fixture_sha256": speech.FIXTURE_SHA,
+              "upstream_commit": speech.UPSTREAM, "candidate": "base", "model_sha256": model_sha,
+              "worker_sha256": WHISPER_SHA, "backend": "forced CPU", "threads": 1, "policy": POLICY,
+              "vad_report_sha256": VAD_SHA, "actual_vad_report": document,
+              "vad_report_utf8": vad_bytes.decode("utf-8"),
+              "harness_sha256": speech.sha256(Path(__file__)),
+              "speech_contract_sha256": speech.sha256(Path(spec.origin)),
+              "ffmpeg_sha256": document["ffmpeg_sha256"], "ffprobe_sha256": document["ffprobe_sha256"],
+              "scope": "Gated ASR diagnostic; no product or word-alignment claim", "rows": []}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.scratch_parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("x", encoding="utf-8") as stream:
+        json.dump(report, stream)
+
+    def checkpoint():
+        args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    with tempfile.TemporaryDirectory(prefix="gated-asr-", dir=args.scratch_parent) as directory:
+        scratch = Path(directory)
+        for detected in vad_rows:
+            asset = assets[detected["asset"]]
+            source = args.fixture.parent / asset["path"]
+            if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]:
+                raise ValueError("Frozen source mismatch")
+            speech.run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
+                "-show_entries", "stream=index,codec_type,duration,start_time", "-of", "json", str(source.resolve())], scratch, "probe", [], 30)
+            probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
+            if probe != detected["probe"]:
+                raise ValueError("Audio probe changed after VAD")
+            start, end, index = speech.primary_audio_range(probe)
+            windows = interval_windows(detected["intervals"], detected["native"]["samples"], start)
+            row = {"asset": asset["id"], "source_sha256": asset["sha256"], "audio_start_ms": start,
+                   "audio_end_ms": end, "duration_ms": end-start, "audio_stream_index": index,
+                   "actual_probe": probe, "status": "speech" if windows else "no_speech",
+                   "asr_not_run": not windows, "windows": []}
+            report["rows"].append(row)
+            checkpoint()
+            for planned in windows:
+                wav, prefix = scratch / "window.wav", scratch / "recognized"
+                output = prefix.with_suffix(".json")
+                output.unlink(missing_ok=True)
+                offset, length = planned["offset_ms"], planned["duration_ms"]
+                decode_ms = speech.run_owned(speech.decode_command(args.ffmpeg, source, offset, length, wav, index), scratch, "decode", [(wav, 970000)], 60)
+                frames = speech.decoded_frames(wav, length)
+                asr_ms = speech.run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
+                    "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
+                    "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, speech.MAX_OUTPUT)], 600)
+                raw = json.loads(output.read_text(encoding="utf-8"))
+                window = speech.recognized_window(raw, offset, length, decode_ms, asr_ms)
+                window.update(planned)
+                window["decoded_frames"] = frames
+                window["actual_asr_log"] = (scratch / "asr.log").read_text(encoding="utf-8")
+                row["windows"].append(window)
+                if "validation_error" in window:
+                    report["validation_failures"].append({"asset": asset["id"], "offset_ms": offset, "error": window["validation_error"]})
+                checkpoint()
+            if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]:
+                raise ValueError("Source changed during recognition")
+    # Reject changed inputs/tools across the entire experiment, including earlier
+    # no-speech rows whose data was verified before later recognition work.
+    for detected in vad_rows:
+        asset = assets[detected["asset"]]
+        source = args.fixture.parent / asset["path"]
+        if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]:
+            raise ValueError("Source changed before comparison completion")
+    if (speech.sha256(args.vad_report) != VAD_SHA or speech.sha256(args.fixture) != speech.FIXTURE_SHA
+            or speech.sha256(args.whisper) != WHISPER_SHA or speech.sha256(args.model) != model_sha
+            or any(speech.sha256(getattr(args, name)) != document[name + "_sha256"] for name in ("ffmpeg", "ffprobe"))):
+        raise ValueError("Experiment tools/provenance changed during recognition")
+    report["complete"] = True
+    report["quality_failed"] = bool(report["validation_failures"])
+    report["comparison_eligible"] = not report["quality_failed"]
+    checkpoint()
+
+
+if __name__ == "__main__":
+    main()

tools/speech-vad/test_compare.py

@@ -0,0 +1,44 @@
+import copy
+import unittest
+from compare import compare, ASSETS
+
+
+class Comparison(unittest.TestCase):
+    def reports(self):
+        common = {"complete": True, "quality_failed": False,
+                  **{key: "same" for key in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads")}}
+        window = {"segments": [{"text": "recognized"}], "decode_ms": 2, "asr_process_ms": 10}
+        baseline = {**common, "rows": [{"asset": asset, "source_sha256": asset, "windows": [copy.deepcopy(window)]} for asset in sorted(ASSETS)]}
+        gated = copy.deepcopy(baseline)
+        gated["comparison_eligible"] = True
+        gated["actual_vad_report"] = {"rows": [{"asset": asset, "decode_ms": 3, "process_ms": 5} for asset in sorted(ASSETS)]}
+        for row in gated["rows"]:
+            row["status"] = "no_speech"
+            row["windows"] = []
+        return baseline, gated
+
+    def test_no_speech_retains_competitors_and_vad_cost(self):
+        baseline, gated = self.reports()
+        result = compare(baseline, gated)
+        self.assertEqual(len(result["rows"]), 5)
+        self.assertEqual(result["rows"][0]["gated"]["asr_process_ms"], 0)
+        self.assertEqual(result["rows"][0]["vad_process_ms"], 5)
+
+    def test_ineligible_or_changed_pool_rejected(self):
+        for mutation in (lambda r: r.update(complete=False), lambda r: r.update(quality_failed=True),
+                         lambda r: r.update(comparison_eligible=False), lambda r: r.update(model_sha256="changed"),
+                         lambda r: r["rows"].pop()):
+            baseline, gated = self.reports()
+            mutation(gated)
+            with self.assertRaises(ValueError):
+                compare(baseline, gated)
+
+    def test_invalid_window_cannot_hide_behind_success(self):
+        baseline, gated = self.reports()
+        baseline["rows"][0]["windows"][0]["validation_error"] = "bad timestamp"
+        with self.assertRaises(ValueError):
+            compare(baseline, gated)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/speech-vad/test_gated_asr.py

@@ -0,0 +1,64 @@
+"""Planning/provenance regressions without loading inference models."""
+import copy
+import json
+from pathlib import Path
+import unittest
+
+import gated_asr as gated
+
+
+class GatedPlanning(unittest.TestCase):
+    def test_absolute_offset_and_contiguous_split(self):
+        interval = gated.vad.source_intersection([10, 6110], 1000000, 5125)
+        windows = gated.interval_windows([interval], 1000000, 5125)
+        self.assertEqual([(w["offset_ms"], w["duration_ms"]) for w in windows],
+                         [(5225, 30000), (35225, 30000), (65225, 1000)])
+        self.assertTrue(all(w["vad_source_intersection"] == interval for w in windows))
+
+    def test_disjoint_intervals_are_not_concatenated(self):
+        intervals = [gated.vad.source_intersection(raw, 16000, 0) for raw in ([0, 20], [80, 90])]
+        self.assertEqual([(w["offset_ms"], w["duration_ms"]) for w in gated.interval_windows(intervals, 16000, 0)],
+                         [(0, 200), (800, 100)])
+
+    def test_tail_fraction_and_raw_padding_remain_explicit(self):
+        interval = gated.vad.source_intersection([90, 102], 16001, 0)
+        window, = gated.interval_windows([interval], 16001, 0)
+        self.assertEqual(window["duration_ms"], 100)
+        self.assertEqual(window["unrepresented_tail_samples"], 1)
+        self.assertEqual(window["vad_source_intersection"]["tail_padding_samples"], 319)
+
+    def test_overlap_and_tampered_mapping_fail(self):
+        intervals = [gated.vad.source_intersection(raw, 16000, 0) for raw in ([0, 50], [40, 60])]
+        with self.assertRaises(ValueError):
+            gated.interval_windows(intervals, 16000, 0)
+        intervals[0]["source_absolute_ms"][0] = 1
+        with self.assertRaises(ValueError):
+            gated.interval_windows(intervals[:1], 16000, 0)
+
+    def test_empty_source_intersection_has_no_asr_window(self):
+        interval = gated.vad.source_intersection([101, 102], 16001, 0)
+        self.assertEqual(gated.interval_windows([interval], 16001, 0), [])
+
+    def test_frozen_real_evidence_is_complete_and_tamper_rejected(self):
+        root = Path(__file__).resolve().parents[2]
+        path = root / ".searchi/speech/vad-v6-results.json"
+        if not path.exists():
+            self.skipTest("Local frozen diagnostic evidence unavailable")
+        self.assertEqual(gated.speech.sha256(path), gated.VAD_SHA)
+        report = json.loads(path.read_bytes())
+        fixture = json.loads((root / "fixtures/media/fixture.json").read_bytes())
+        self.assertEqual(len(gated.validated_rows(report, fixture)), 5)
+        for mutate in (
+            lambda r: r["rows"].pop(),
+            lambda r: r["rows"][0]["native"].update(gpu=True),
+            lambda r: r["rows"][0]["native"]["probabilities"].pop(),
+            lambda r: r["rows"][0].update(source_sha256="bad"),
+        ):
+            changed = copy.deepcopy(report)
+            mutate(changed)
+            with self.assertRaises(ValueError):
+                gated.validated_rows(changed, fixture)
+
+
+if __name__ == "__main__":
+    unittest.main()