From af04cec27c2eab720eb98b8220052f843e213f05 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 13:47:23 +0200 Subject: [PATCH] Measure complete CPU gated transcription and component costs --- docs/SPEECH-VAD.md | 26 +++++ tools/speech-vad/README.md | 6 + tools/speech-vad/compare.py | 47 ++++++++ tools/speech-vad/gated_asr.py | 170 +++++++++++++++++++++++++++++ tools/speech-vad/test_compare.py | 44 ++++++++ tools/speech-vad/test_gated_asr.py | 64 +++++++++++ 6 files changed, 357 insertions(+) create mode 100644 tools/speech-vad/compare.py create mode 100644 tools/speech-vad/gated_asr.py create mode 100644 tools/speech-vad/test_compare.py create mode 100644 tools/speech-vad/test_gated_asr.py diff --git a/docs/SPEECH-VAD.md b/docs/SPEECH-VAD.md index 81096d6..948260b 100644 --- a/docs/SPEECH-VAD.md +++ b/docs/SPEECH-VAD.md @@ -222,6 +222,32 @@ speech recall, transcript correctness, timestamp accuracy, retrieval improvement or a general music policy. No production gate was added and original Google audio retrieval remains unchanged. +## Actual gated base transcription + +The pinned base ASR worker transcribed the twelve original contiguous speech +intervals on CPU with one thread, unchanged auto-language decoding flags, and +no integrated VAD concatenation. All five competitor rows remain present; +four explicit no-speech rows launch no ASR. All timestamps and decoded coverage +passed validation. Raw ASR, logs and embedded frozen VAD evidence are retained +in `.searchi/speech/gated-base-results.json`, SHA-256 +`f2af1e650af697e4540cbc3e90c0035ff560cb675186f5dad9b2afff01e53a56`. +Six harness tests and three timing-comparison tests passed. + +The sequential sum of decode plus ASR process times was 250,762.6 ms in the +ungated baseline. Gated decode plus ASR **including the earlier VAD decode and +process cost** totaled 64,912.2 ms. Probe/hash/checkpoint time is not included, +these are separate runs, and this is not search latency or a general speedup. +Speech alone became slower: ASR process time rose from 23,321.7 to 58,629.4 ms +because twelve independent model starts replaced three. It yielded twenty +segments versus nineteen; segment count does not prove transcript accuracy. +Music/sound ASR text vanished through explicit no-speech decisions, not manual +phrase filtering. Original Google audio retrieval remains independent. + +The next gate is genuine Google retrieval on all eight unchanged queries and +independent speech recall/alignment assessment. Do not select a production +policy from timing or this tiny corpus alone. A future persistent ASR session +could reduce model-start overhead but must be measured independently. + Open gates: actual one-thread audit, detection/recognition comparison, ordinary and DTW timestamp regression, independent held-out quality, background starvation/cancellation/crash recovery, packaging/license review, diff --git a/tools/speech-vad/README.md b/tools/speech-vad/README.md index ef15cff..7fe88fd 100644 --- a/tools/speech-vad/README.md +++ b/tools/speech-vad/README.md @@ -30,3 +30,9 @@ minimum speech 250 ms, silence 100 ms, maximum speech FLT_MAX, padding 30 ms, overlap 0.1 seconds. It does not establish model accuracy or a production speech gate. Compare the complete frozen speech/music/sound pool before judging quality; preserve original Google audio retrieval regardless of transcription. + +`gated_asr.py` runs the frozen base transcription experiment from the verified +VAD report. It preserves all five competitors, no-speech status, original source +intervals and raw ASR logs. `compare.py BASELINE GATED` reports component timing +and segment counts only; it rejects incomplete or invalid pools and does not +score retrieval quality. See the protocol document for actual evidence. diff --git a/tools/speech-vad/compare.py b/tools/speech-vad/compare.py new file mode 100644 index 0000000..4507440 --- /dev/null +++ b/tools/speech-vad/compare.py @@ -0,0 +1,47 @@ +"""Compare complete frozen CPU ASR pools; timing is not retrieval quality.""" +import argparse +import json +from pathlib import Path + +ASSETS = {"clip04", "music01", "music02", "sound01", "sound02"} + + +def compare(baseline, gated): + for report in (baseline, gated): + if report.get("complete") is not True or report.get("quality_failed") is not False: + raise ValueError("Incomplete or structurally invalid comparison pool") + rows = report.get("rows", []) + if len(rows) != 5 or {row["asset"] for row in rows} != ASSETS: + raise ValueError("All five unique competitors required") + for field in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads"): + if baseline[field] != gated[field]: + raise ValueError("Comparison provenance differs: " + field) + if gated.get("comparison_eligible") is not True: + raise ValueError("Gated comparison is ineligible") + original = {row["asset"]: row for row in baseline["rows"]} + vad_rows = {row["asset"]: row for row in gated["actual_vad_report"]["rows"]} + results = [] + for row in gated["rows"]: + old = original[row["asset"]] + if old["source_sha256"] != row["source_sha256"]: + raise ValueError("Changed source") + def stats(windows): + if any("validation_error" in window for window in windows): + raise ValueError("Invalid window must not be scored") + return {"windows": len(windows), "segments": sum(len(w["segments"]) for w in windows), + "decode_ms": sum(w["decode_ms"] for w in windows), + "asr_process_ms": sum(w["asr_process_ms"] for w in windows)} + results.append({"asset": row["asset"], "baseline": stats(old["windows"]), + "gated": stats(row["windows"]), "status": row["status"], + "vad_decode_ms": vad_rows[row["asset"]]["decode_ms"], + "vad_process_ms": vad_rows[row["asset"]]["process_ms"]}) + return {"scope": "Small sequential CPU diagnostic; process timings include model loading; not search latency or quality", "rows": results} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path) + parser.add_argument("gated", type=Path) + args = parser.parse_args() + print(json.dumps(compare(json.loads(args.baseline.read_text(encoding="utf-8")), + json.loads(args.gated.read_text(encoding="utf-8"))), indent=2)) diff --git a/tools/speech-vad/gated_asr.py b/tools/speech-vad/gated_asr.py new file mode 100644 index 0000000..ef1246f --- /dev/null +++ b/tools/speech-vad/gated_asr.py @@ -0,0 +1,170 @@ +"""Frozen VAD-gated CPU ASR diagnostic; never changes product search.""" +import argparse +import importlib.util +import json +import math +from pathlib import Path +import tempfile + +spec = importlib.util.spec_from_file_location("gated_speech_contract", Path(__file__).parents[1] / "speech-eval/run.py") +speech = importlib.util.module_from_spec(spec) +spec.loader.exec_module(speech) +vad_spec = importlib.util.spec_from_file_location("gated_vad_contract", Path(__file__).with_name("run.py")) +vad = importlib.util.module_from_spec(vad_spec) +vad_spec.loader.exec_module(vad) +VAD_SHA = "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df" +VAD_WORKER_SHA = "28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19" +WHISPER_SHA = "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8" +POLICY = "gated-vad-v1;source-intersections;integer-ms-tail-remainder-recorded;contiguous30s;independent-process;auto-language;segment-times" + + +def interval_windows(intervals, samples, origin): + """Never concatenate intervals or reapply padding; retain sub-ms tail loss.""" + result = [] + previous = 0 + for index, interval in enumerate(intervals): + expected = vad.source_intersection(interval["raw_cs"], samples, origin) + if interval != expected: + raise ValueError("VAD source intersection differs from raw evidence") + start, end = interval["source_samples"] + if start < previous: + raise ValueError("Overlapping or unordered source intersections") + previous = end + # Raw starts are integer centiseconds; only the final bounded tail can + # have a sample fraction that whisper's integer-ms serializer cannot hold. + first, last = origin + start // 16, origin + end // 16 + for offset in range(first, last, 30000): + result.append({"offset_ms": offset, "duration_ms": min(30000, last - offset), + "vad_interval_index": index, "vad_source_intersection": interval, + "unrepresented_tail_samples": end % 16}) + return result + + +def validated_rows(document, fixture): + if (document.get("schema") != 1 or document.get("complete") is not True + or document.get("fixture_sha256") != speech.FIXTURE_SHA + or document.get("model_sha256") != vad.MODEL_SHA + or document.get("worker_sha256") != VAD_WORKER_SHA): + raise ValueError("Unrecognized frozen VAD provenance") + rows = document.get("rows", []) + if sorted(row["asset"] for row in rows) != sorted(speech.ASSETS): + raise ValueError("Incomplete or duplicate VAD competitor pool") + assets = {asset["id"]: asset for asset in fixture["assets"]} + for row in rows: + if row["source_sha256"] != assets[row["asset"]]["sha256"]: + raise ValueError("VAD source hash mismatch") + native = row["native"] + start, end, _ = speech.primary_audio_range(row["probe"]) + samples = native["samples"] + probabilities = native["probabilities"] + if (type(samples) is not int or samples <= 0 or abs(samples - (end-start)*16) > 16 + or native["sample_rate"] != 16000 or native["threads"] != 1 or native["gpu"] is not False + or len(probabilities) != (samples + 511) // 512 + or any(type(p) not in (int, float) or not math.isfinite(p) or not 0 <= p <= 1 for p in probabilities) + or [interval["raw_cs"] for interval in row["intervals"]] != native["intervals_cs"]): + raise ValueError("Invalid VAD native coverage or CPU contract") + interval_windows(row["intervals"], samples, start) + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for option in ("whisper", "model", "ffmpeg", "ffprobe", "vad-report", "scratch-parent", "output"): + parser.add_argument("--" + option, type=Path, required=True) + parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json")) + args = parser.parse_args() + if args.output.exists(): + raise ValueError("Never overwrite experiment evidence") + if speech.sha256(args.vad_report) != VAD_SHA or speech.sha256(args.fixture) != speech.FIXTURE_SHA: + raise ValueError("Frozen VAD/fixture SHA mismatch") + size, model_sha = speech.MODELS["base"] + if args.model.stat().st_size != size or speech.sha256(args.model) != model_sha or speech.sha256(args.whisper) != WHISPER_SHA: + raise ValueError("Frozen ASR worker/model mismatch") + if args.vad_report.stat().st_size > 16 * 1024 * 1024: + raise ValueError("Oversized VAD evidence") + vad_bytes = args.vad_report.read_bytes() + document = json.loads(vad_bytes) + for name in ("ffmpeg", "ffprobe"): + if speech.sha256(getattr(args, name)) != document[name + "_sha256"]: + raise ValueError("Decoder provenance differs from frozen VAD") + fixture = json.loads(args.fixture.read_text(encoding="utf-8")) + vad_rows = validated_rows(document, fixture) + assets = {asset["id"]: asset for asset in fixture["assets"]} + report = {"version": 1, "complete": False, "quality_failed": False, "validation_failures": [], + "comparison_eligible": False, "fixture_sha256": speech.FIXTURE_SHA, + "upstream_commit": speech.UPSTREAM, "candidate": "base", "model_sha256": model_sha, + "worker_sha256": WHISPER_SHA, "backend": "forced CPU", "threads": 1, "policy": POLICY, + "vad_report_sha256": VAD_SHA, "actual_vad_report": document, + "vad_report_utf8": vad_bytes.decode("utf-8"), + "harness_sha256": speech.sha256(Path(__file__)), + "speech_contract_sha256": speech.sha256(Path(spec.origin)), + "ffmpeg_sha256": document["ffmpeg_sha256"], "ffprobe_sha256": document["ffprobe_sha256"], + "scope": "Gated ASR diagnostic; no product or word-alignment claim", "rows": []} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.scratch_parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x", encoding="utf-8") as stream: + json.dump(report, stream) + + def checkpoint(): + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + with tempfile.TemporaryDirectory(prefix="gated-asr-", dir=args.scratch_parent) as directory: + scratch = Path(directory) + for detected in vad_rows: + asset = assets[detected["asset"]] + source = args.fixture.parent / asset["path"] + if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]: + raise ValueError("Frozen source mismatch") + speech.run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe", + "-show_entries", "stream=index,codec_type,duration,start_time", "-of", "json", str(source.resolve())], scratch, "probe", [], 30) + probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8")) + if probe != detected["probe"]: + raise ValueError("Audio probe changed after VAD") + start, end, index = speech.primary_audio_range(probe) + windows = interval_windows(detected["intervals"], detected["native"]["samples"], start) + row = {"asset": asset["id"], "source_sha256": asset["sha256"], "audio_start_ms": start, + "audio_end_ms": end, "duration_ms": end-start, "audio_stream_index": index, + "actual_probe": probe, "status": "speech" if windows else "no_speech", + "asr_not_run": not windows, "windows": []} + report["rows"].append(row) + checkpoint() + for planned in windows: + wav, prefix = scratch / "window.wav", scratch / "recognized" + output = prefix.with_suffix(".json") + output.unlink(missing_ok=True) + offset, length = planned["offset_ms"], planned["duration_ms"] + decode_ms = speech.run_owned(speech.decode_command(args.ffmpeg, source, offset, length, wav, index), scratch, "decode", [(wav, 970000)], 60) + frames = speech.decoded_frames(wav, length) + asr_ms = speech.run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1", + "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav), + "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, speech.MAX_OUTPUT)], 600) + raw = json.loads(output.read_text(encoding="utf-8")) + window = speech.recognized_window(raw, offset, length, decode_ms, asr_ms) + window.update(planned) + window["decoded_frames"] = frames + window["actual_asr_log"] = (scratch / "asr.log").read_text(encoding="utf-8") + row["windows"].append(window) + if "validation_error" in window: + report["validation_failures"].append({"asset": asset["id"], "offset_ms": offset, "error": window["validation_error"]}) + checkpoint() + if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]: + raise ValueError("Source changed during recognition") + # Reject changed inputs/tools across the entire experiment, including earlier + # no-speech rows whose data was verified before later recognition work. + for detected in vad_rows: + asset = assets[detected["asset"]] + source = args.fixture.parent / asset["path"] + if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]: + raise ValueError("Source changed before comparison completion") + if (speech.sha256(args.vad_report) != VAD_SHA or speech.sha256(args.fixture) != speech.FIXTURE_SHA + or speech.sha256(args.whisper) != WHISPER_SHA or speech.sha256(args.model) != model_sha + or any(speech.sha256(getattr(args, name)) != document[name + "_sha256"] for name in ("ffmpeg", "ffprobe"))): + raise ValueError("Experiment tools/provenance changed during recognition") + report["complete"] = True + report["quality_failed"] = bool(report["validation_failures"]) + report["comparison_eligible"] = not report["quality_failed"] + checkpoint() + + +if __name__ == "__main__": + main() diff --git a/tools/speech-vad/test_compare.py b/tools/speech-vad/test_compare.py new file mode 100644 index 0000000..6130498 --- /dev/null +++ b/tools/speech-vad/test_compare.py @@ -0,0 +1,44 @@ +import copy +import unittest +from compare import compare, ASSETS + + +class Comparison(unittest.TestCase): + def reports(self): + common = {"complete": True, "quality_failed": False, + **{key: "same" for key in ("fixture_sha256", "model_sha256", "worker_sha256", "ffmpeg_sha256", "ffprobe_sha256", "backend", "threads")}} + window = {"segments": [{"text": "recognized"}], "decode_ms": 2, "asr_process_ms": 10} + baseline = {**common, "rows": [{"asset": asset, "source_sha256": asset, "windows": [copy.deepcopy(window)]} for asset in sorted(ASSETS)]} + gated = copy.deepcopy(baseline) + gated["comparison_eligible"] = True + gated["actual_vad_report"] = {"rows": [{"asset": asset, "decode_ms": 3, "process_ms": 5} for asset in sorted(ASSETS)]} + for row in gated["rows"]: + row["status"] = "no_speech" + row["windows"] = [] + return baseline, gated + + def test_no_speech_retains_competitors_and_vad_cost(self): + baseline, gated = self.reports() + result = compare(baseline, gated) + self.assertEqual(len(result["rows"]), 5) + self.assertEqual(result["rows"][0]["gated"]["asr_process_ms"], 0) + self.assertEqual(result["rows"][0]["vad_process_ms"], 5) + + def test_ineligible_or_changed_pool_rejected(self): + for mutation in (lambda r: r.update(complete=False), lambda r: r.update(quality_failed=True), + lambda r: r.update(comparison_eligible=False), lambda r: r.update(model_sha256="changed"), + lambda r: r["rows"].pop()): + baseline, gated = self.reports() + mutation(gated) + with self.assertRaises(ValueError): + compare(baseline, gated) + + def test_invalid_window_cannot_hide_behind_success(self): + baseline, gated = self.reports() + baseline["rows"][0]["windows"][0]["validation_error"] = "bad timestamp" + with self.assertRaises(ValueError): + compare(baseline, gated) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/speech-vad/test_gated_asr.py b/tools/speech-vad/test_gated_asr.py new file mode 100644 index 0000000..25dca36 --- /dev/null +++ b/tools/speech-vad/test_gated_asr.py @@ -0,0 +1,64 @@ +"""Planning/provenance regressions without loading inference models.""" +import copy +import json +from pathlib import Path +import unittest + +import gated_asr as gated + + +class GatedPlanning(unittest.TestCase): + def test_absolute_offset_and_contiguous_split(self): + interval = gated.vad.source_intersection([10, 6110], 1000000, 5125) + windows = gated.interval_windows([interval], 1000000, 5125) + self.assertEqual([(w["offset_ms"], w["duration_ms"]) for w in windows], + [(5225, 30000), (35225, 30000), (65225, 1000)]) + self.assertTrue(all(w["vad_source_intersection"] == interval for w in windows)) + + def test_disjoint_intervals_are_not_concatenated(self): + intervals = [gated.vad.source_intersection(raw, 16000, 0) for raw in ([0, 20], [80, 90])] + self.assertEqual([(w["offset_ms"], w["duration_ms"]) for w in gated.interval_windows(intervals, 16000, 0)], + [(0, 200), (800, 100)]) + + def test_tail_fraction_and_raw_padding_remain_explicit(self): + interval = gated.vad.source_intersection([90, 102], 16001, 0) + window, = gated.interval_windows([interval], 16001, 0) + self.assertEqual(window["duration_ms"], 100) + self.assertEqual(window["unrepresented_tail_samples"], 1) + self.assertEqual(window["vad_source_intersection"]["tail_padding_samples"], 319) + + def test_overlap_and_tampered_mapping_fail(self): + intervals = [gated.vad.source_intersection(raw, 16000, 0) for raw in ([0, 50], [40, 60])] + with self.assertRaises(ValueError): + gated.interval_windows(intervals, 16000, 0) + intervals[0]["source_absolute_ms"][0] = 1 + with self.assertRaises(ValueError): + gated.interval_windows(intervals[:1], 16000, 0) + + def test_empty_source_intersection_has_no_asr_window(self): + interval = gated.vad.source_intersection([101, 102], 16001, 0) + self.assertEqual(gated.interval_windows([interval], 16001, 0), []) + + def test_frozen_real_evidence_is_complete_and_tamper_rejected(self): + root = Path(__file__).resolve().parents[2] + path = root / ".searchi/speech/vad-v6-results.json" + if not path.exists(): + self.skipTest("Local frozen diagnostic evidence unavailable") + self.assertEqual(gated.speech.sha256(path), gated.VAD_SHA) + report = json.loads(path.read_bytes()) + fixture = json.loads((root / "fixtures/media/fixture.json").read_bytes()) + self.assertEqual(len(gated.validated_rows(report, fixture)), 5) + for mutate in ( + lambda r: r["rows"].pop(), + lambda r: r["rows"][0]["native"].update(gpu=True), + lambda r: r["rows"][0]["native"]["probabilities"].pop(), + lambda r: r["rows"][0].update(source_sha256="bad"), + ): + changed = copy.deepcopy(report) + mutate(changed) + with self.assertRaises(ValueError): + gated.validated_rows(changed, fixture) + + +if __name__ == "__main__": + unittest.main()