diff --git a/docs/SPEECH-VAD.md b/docs/SPEECH-VAD.md index 4879e98..e41cab5 100644 --- a/docs/SPEECH-VAD.md +++ b/docs/SPEECH-VAD.md @@ -2,7 +2,8 @@ Research recorded 2026-10-13 against whisper.cpp **v1.9.5**, exact commit `d1be6fde11ac6e0407606b4e42fe72d34add8037`. A real-model CPU silence contract -check has now passed; no media comparison, alignment experiment, product +check and complete five-source VAD run have now passed; no gated ASR comparison, +alignment experiment, product integration or default change has been performed. The corrected ungated ASR observations remain in [speech alignment](SPEECH-ALIGNMENT.md). The instrumental piano's large CPU cost and unrelated text motivate testing; @@ -183,6 +184,38 @@ parameter table, catches native error-level logs, and checks probability frame coverage. This is a smoke test, not a speech-quality result or OS thread-count measurement. An independent complete media comparison remains necessary. +## Complete frozen-pool VAD measurement + +The tracked `tools/speech-vad/run.py` verified every source hash and primary +audio stream, decoded float PCM with full sample coverage, and ran the v6.2.0 +candidate over all five existing competitors. Raw probabilities/logs and both +raw and source-intersected intervals are retained in ignored +`.searchi/speech/vad-v6-results.json`, SHA-256 +`093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df`. +Current diagnostic worker SHA-256: +`28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19`. + +| Source | Intervals | Detected source seconds | Whole VAD process ms | +| --- | ---: | ---: | ---: | +| clip04 (speech) | 12 | 66.04 | 2002.7 | +| music01 | 0 | 0 | 870.1 | +| music02 | 0 | 0 | 160.1 | +| sound01 | 0 | 0 | 57.7 | +| sound02 | 0 | 0 | 108.3 | + +The first attempted run rejected finite float decoder overshoot above unity. +That input contract was incorrect: float decoding can exceed unity without +corruption. The wrapper now preserves finite samples without clipping and +rejects nonfinite values. The failure log was retained and no reduced pool +was scored. All three native contracts passed again with the corrected build; +four mapping regressions cover centiseconds, source origin, padded tails and +invalid intervals. Segmentation parameters and model remained frozen. + +This small VAD-only result supports testing gated ASR next. It does not establish +speech recall, transcript correctness, timestamp accuracy, retrieval improvement +or a general music policy. No production gate was added and original Google +audio retrieval remains unchanged. + Open gates: actual one-thread audit, detection/recognition comparison, ordinary and DTW timestamp regression, independent held-out quality, background starvation/cancellation/crash recovery, packaging/license review, diff --git a/tools/speech-vad/README.md b/tools/speech-vad/README.md index 34ef61c..ef15cff 100644 --- a/tools/speech-vad/README.md +++ b/tools/speech-vad/README.md @@ -12,7 +12,8 @@ cmake --build .searchi/speech/build-vad-diagnostic --config Release --parallel 1 Run `searchi-vad-diagnostic MODEL PCM_F32LE`, retaining stdout and stderr separately. Independently verify the model SHA-256 before running. Input must be verified mono 16-kHz little-endian float32 PCM, at most ten minutes, with -finite normalized samples. CPU and one inference thread are fixed. Native error +finite samples. Decoder overshoot beyond unity is preserved without clipping. +CPU and one inference thread are fixed. Native error logs invalidate detection even when upstream returns success. Probability count must match the pinned 512-sample inference framing. diff --git a/tools/speech-vad/diagnostic.cpp b/tools/speech-vad/diagnostic.cpp index c75f217..ecdb113 100644 --- a/tools/speech-vad/diagnostic.cpp +++ b/tools/speech-vad/diagnostic.cpp @@ -31,7 +31,9 @@ int main(int argc, char ** argv) { std::vector samples(static_cast(bytes) / 4); input.seekg(0); if (!input.read(reinterpret_cast(samples.data()), bytes)) return 2; - for (float sample : samples) if (!std::isfinite(sample) || std::abs(sample) > 1.0f) return 2; + // Float decoding may legitimately overshoot unity. Preserve source samples; + // clipping would change the experiment. Reject nonfinite input only. + for (float sample : samples) if (!std::isfinite(sample)) return 2; whisper_log_set(log_message, nullptr); auto context_params = whisper_vad_default_context_params(); context_params.n_threads = 1; diff --git a/tools/speech-vad/run.py b/tools/speech-vad/run.py new file mode 100644 index 0000000..6d8f80a --- /dev/null +++ b/tools/speech-vad/run.py @@ -0,0 +1,85 @@ +"""Frozen complete media-pool CPU VAD experiment; no ASR or ranking changes.""" +import argparse +import importlib.util +import json +from pathlib import Path +import tempfile + +spec = importlib.util.spec_from_file_location("speech_contract", Path(__file__).parents[1] / "speech-eval/run.py") +speech = importlib.util.module_from_spec(spec) +spec.loader.exec_module(speech) +MODEL_SHA = "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987" + + +def source_intersection(interval, samples, origin_ms): + start, end = interval + if start < 0 or end <= start or int(start) != start or int(end) != end: + raise ValueError("Invalid raw centisecond interval") + raw_start, raw_end = int(start) * 160, int(end) * 160 + bounded_start, bounded_end = min(raw_start, samples), min(raw_end, samples) + return {"raw_cs": interval, "source_samples": [bounded_start, bounded_end], + "source_absolute_ms": [origin_ms + bounded_start / 16, origin_ms + bounded_end / 16], + "tail_padding_samples": max(0, raw_end - samples)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for option in ("worker", "model", "ffmpeg", "ffprobe", "output", "scratch-parent"): + parser.add_argument("--" + option, type=Path, required=True) + parser.add_argument("--worker-sha256", required=True) + parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json")) + args = parser.parse_args() + if args.output.exists(): + raise ValueError("Never overwrite experiment evidence") + if speech.sha256(args.worker) != args.worker_sha256 or speech.sha256(args.model) != MODEL_SHA: + raise ValueError("Frozen worker/model mismatch") + if speech.sha256(args.fixture) != speech.FIXTURE_SHA: + raise ValueError("Frozen media fixture mismatch") + fixture = json.loads(args.fixture.read_text(encoding="utf-8")) + selected = [asset for asset in fixture["assets"] if asset["id"] in speech.ASSETS] + if sorted(asset["id"] for asset in selected) != sorted(speech.ASSETS): + raise ValueError("Incomplete or duplicate competitor pool") + rows = [] + args.scratch_parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="vad-proof-", dir=args.scratch_parent) as scratch_dir: + scratch = Path(scratch_dir) + for asset in selected: + source = args.fixture.parent / asset["path"] + if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]: + raise ValueError("Frozen source mismatch") + probe_ms = speech.run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe", + "-show_entries", "stream=index,codec_type,duration,start_time", "-of", "json", str(source.resolve())], scratch, "probe", [], 30) + probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8")) + start, end, index = speech.primary_audio_range(probe) + pcm = scratch / "audio.f32" + command = speech.decode_command(args.ffmpeg, source, start, end - start, pcm, index) + command[command.index("pcm_s16le")] = "pcm_f32le" + command[-1:-1] = ["-f", "f32le"] + decode_ms = speech.run_owned(command, scratch, "decode", [(pcm, 16000 * 600 * 4)], 60) + samples = pcm.stat().st_size // 4 + if pcm.stat().st_size % 4 or abs(samples - (end - start) * 16) > 16: + raise ValueError("Incomplete decoded coverage") + process_ms = speech.run_owned([str(args.worker.resolve()), str(args.model.resolve()), str(pcm)], scratch, "vad", [], 120) + log = (scratch / "vad.log").read_text(encoding="utf-8") + report = json.loads(log.strip().splitlines()[-1]) + if report["samples"] != samples or report["threads"] != 1 or report["gpu"] is not False: + raise ValueError("Unexpected native contract") + if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]: + raise ValueError("Source changed during detection") + rows.append({"asset": asset["id"], "source_sha256": asset["sha256"], "probe": probe, + "probe_ms": probe_ms, "decode_ms": decode_ms, "process_ms": process_ms, + "native": report, "native_log": log, + "intervals": [source_intersection(interval, samples, start) for interval in report["intervals_cs"]]}) + result = {"schema": 1, "complete": True, "scope": "VAD only; no ASR quality or retrieval claim", + "fixture_sha256": speech.FIXTURE_SHA, "worker_sha256": args.worker_sha256, + "model_sha256": MODEL_SHA, "harness_sha256": speech.sha256(Path(__file__)), + "speech_contract_sha256": speech.sha256(Path(spec.origin)), + "ffmpeg_sha256": speech.sha256(args.ffmpeg), "ffprobe_sha256": speech.sha256(args.ffprobe), "rows": rows} + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x", encoding="utf-8") as stream: + json.dump(result, stream, indent=2) + stream.write("\n") + + +if __name__ == "__main__": + main() diff --git a/tools/speech-vad/test_mapping.py b/tools/speech-vad/test_mapping.py new file mode 100644 index 0000000..975198f --- /dev/null +++ b/tools/speech-vad/test_mapping.py @@ -0,0 +1,27 @@ +import unittest +from run import source_intersection + + +class Mapping(unittest.TestCase): + def test_units_and_nonzero_origin_once(self): + interval = source_intersection([10, 25], 16000, 5125) + self.assertEqual(interval["source_samples"], [1600, 4000]) + self.assertEqual(interval["source_absolute_ms"], [5225, 5375]) + + def test_padded_tail_keeps_raw_evidence(self): + interval = source_intersection([90, 102], 16001, 0) + self.assertEqual(interval["raw_cs"], [90, 102]) + self.assertEqual(interval["source_samples"], [14400, 16001]) + self.assertEqual(interval["tail_padding_samples"], 319) + + def test_synthetic_only_interval_has_empty_source_intersection(self): + self.assertEqual(source_intersection([101, 102], 16001, 0)["source_samples"], [16001, 16001]) + + def test_bad_units_and_order_fail(self): + for interval in ([0.1, 2], [-1, 2], [2, 2], [3, 2]): + with self.subTest(interval=interval), self.assertRaises(ValueError): + source_intersection(interval, 16000, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/speech-vad/test_native.py b/tools/speech-vad/test_native.py index aae75f1..d225504 100644 --- a/tools/speech-vad/test_native.py +++ b/tools/speech-vad/test_native.py @@ -37,7 +37,7 @@ class NativeContract(unittest.TestCase): self.assertGreaterEqual(report[key], 0) def test_invalid_pcm(self): - for data in (b"", b"x", struct.pack("