diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 9315331..3843beb 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -79,6 +79,9 @@ release. Keep their evidence and unchecked production gates intact. calibrated handling of queries without a relevant match. - [x] Prepare bounded CPU speech diagnostics and test process failures, timestamp units and frozen-label scoring; see [speech proposal](SPEECH-ALIGNMENT.md). + - [x] Build a pinned CPU VAD diagnostic with explicit one-thread inference, + native error detection and full probabilities; actual Silero silence/input + contracts passed. Media quality comparison remains open (see SPEECH-VAD.md). - [ ] Run genuine local speech candidates, Google transcript retrieval and held-out English/German/music/no-speech checks before selecting a default. - [ ] Package and verify decoders on every release OS; test production acceleration. diff --git a/docs/SPEECH-VAD.md b/docs/SPEECH-VAD.md index c056cf1..4879e98 100644 --- a/docs/SPEECH-VAD.md +++ b/docs/SPEECH-VAD.md @@ -1,8 +1,9 @@ # Speech gating and timestamp research Research recorded 2026-10-13 against whisper.cpp **v1.9.5**, exact commit -`d1be6fde11ac6e0407606b4e42fe72d34add8037`. No VAD inference, alignment -experiment, product integration or default change has been performed. The +`d1be6fde11ac6e0407606b4e42fe72d34add8037`. A real-model CPU silence contract +check has now passed; no media comparison, alignment experiment, product +integration or default change has been performed. The corrected ungated ASR observations remain in [speech alignment](SPEECH-ALIGNMENT.md). The instrumental piano's large CPU cost and unrelated text motivate testing; they do not establish that any particular VAD model fixes the problem. @@ -171,6 +172,17 @@ Source review identified further requirements before accepting measurements: mode and one thread, check segment allocation, retain probabilities and separate model loading, detection and segmentation timings. +The separately tracked [diagnostic wrapper](../tools/speech-vad/README.md) also +built successfully. Its SHA-256 is +`095a05c85c6f8499608ba8caa46b7dc3bbc0258a8585eeecec74ccee4b3e19d4`. +Three native contract tests passed in 1.451 seconds using the verified v6.2.0 +model: 16,001 silent samples yielded all 32 expected probabilities and no speech +intervals; empty/misaligned/nonfinite/out-of-range PCM and unknown arguments +failed without JSON. The wrapper fixes CPU mode, one inference thread and the +parameter table, catches native error-level logs, and checks probability frame +coverage. This is a smoke test, not a speech-quality result or OS thread-count +measurement. An independent complete media comparison remains necessary. + Open gates: actual one-thread audit, detection/recognition comparison, ordinary and DTW timestamp regression, independent held-out quality, background starvation/cancellation/crash recovery, packaging/license review, diff --git a/tools/speech-vad/CMakeLists.txt b/tools/speech-vad/CMakeLists.txt new file mode 100644 index 0000000..06b1094 --- /dev/null +++ b/tools/speech-vad/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.21) +project(searchi_vad_diagnostic LANGUAGES C CXX) +if(NOT WHISPER_SOURCE) + message(FATAL_ERROR "Provide the pinned WHISPER_SOURCE checkout") +endif() +find_package(Git REQUIRED) +execute_process(COMMAND "${GIT_EXECUTABLE}" -C "${WHISPER_SOURCE}" rev-parse HEAD + OUTPUT_VARIABLE whisper_revision OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE revision_status) +if(NOT revision_status EQUAL 0 OR NOT whisper_revision STREQUAL "d1be6fde11ac6e0407606b4e42fe72d34add8037") + message(FATAL_ERROR "Diagnostic requires the frozen whisper.cpp revision") +endif() +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE) +foreach(provider CUDA VULKAN METAL SYCL HIP BLAS) + set(GGML_${provider} OFF CACHE BOOL "" FORCE) +endforeach() +set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE) +add_subdirectory("${WHISPER_SOURCE}" whisper) +add_executable(searchi-vad-diagnostic diagnostic.cpp) +target_compile_features(searchi-vad-diagnostic PRIVATE cxx_std_17) +target_link_libraries(searchi-vad-diagnostic PRIVATE whisper) diff --git a/tools/speech-vad/README.md b/tools/speech-vad/README.md new file mode 100644 index 0000000..34ef61c --- /dev/null +++ b/tools/speech-vad/README.md @@ -0,0 +1,31 @@ +# CPU speech-activity diagnostic + +Research tooling, separate from searchi's normal GPU-capable runtime. No model +download, transcription, source indexing or product default is changed. +Use the frozen source and model hashes in [the protocol](../../docs/SPEECH-VAD.md). + +```sh +cmake -S tools/speech-vad -B .searchi/speech/build-vad-diagnostic -DWHISPER_SOURCE=/absolute/path/to/pinned/whisper.cpp +cmake --build .searchi/speech/build-vad-diagnostic --config Release --parallel 1 +``` + +Run `searchi-vad-diagnostic MODEL PCM_F32LE`, retaining stdout and stderr +separately. Independently verify the model SHA-256 before running. Input must +be verified mono 16-kHz little-endian float32 PCM, at most ten minutes, with +finite normalized samples. CPU and one inference thread are fixed. Native error +logs invalidate detection even when upstream returns success. Probability +count must match the pinned 512-sample inference framing. + +Output JSON retains every probability and raw centisecond interval, source +sample count and load/detection/segmentation milliseconds. Interval padding is +already applied. Preserve this raw output; a subsequent harness must explicitly +intersect intervals with actual decoded source samples and add the verified +source origin exactly once. Do not concatenate disjoint intervals without a +piecewise source map. Zero intervals are a valid no-speech decision, distinct +from nonzero exit status or missing JSON. + +The wrapper fixes the preregistered segmentation parameters: threshold 0.5, +minimum speech 250 ms, silence 100 ms, maximum speech FLT_MAX, padding 30 ms, +overlap 0.1 seconds. It does not establish model accuracy or a production +speech gate. Compare the complete frozen speech/music/sound pool before judging +quality; preserve original Google audio retrieval regardless of transcription. diff --git a/tools/speech-vad/diagnostic.cpp b/tools/speech-vad/diagnostic.cpp new file mode 100644 index 0000000..c75f217 --- /dev/null +++ b/tools/speech-vad/diagnostic.cpp @@ -0,0 +1,81 @@ +// Research only. Input: verified mono 16-kHz little-endian float32 PCM. +#include "whisper.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::atomic failed{false}; +static void log_message(enum ggml_log_level level, const char * text, void *) { + if (level == GGML_LOG_LEVEL_ERROR) failed.store(true); + std::fputs(text, stderr); +} +using Clock = std::chrono::steady_clock; +static double elapsed(Clock::time_point start) { + return std::chrono::duration(Clock::now() - start).count(); +} +int main(int argc, char ** argv) { + if (argc != 3) { std::cerr << "Expected MODEL PCM_F32LE\n"; return 2; } + const unsigned short endian = 1; + if (*reinterpret_cast(&endian) != 1 || sizeof(float) != 4) return 2; + std::ifstream input(argv[2], std::ios::binary | std::ios::ate); + if (!input) return 2; + const auto bytes = input.tellg(); + // At most ten minutes. Never allocate according to an unchecked file size. + if (bytes <= 0 || bytes > 16000LL * 600 * 4 || bytes % 4 != 0) return 2; + std::vector samples(static_cast(bytes) / 4); + input.seekg(0); + if (!input.read(reinterpret_cast(samples.data()), bytes)) return 2; + for (float sample : samples) if (!std::isfinite(sample) || std::abs(sample) > 1.0f) return 2; + whisper_log_set(log_message, nullptr); + auto context_params = whisper_vad_default_context_params(); + context_params.n_threads = 1; + context_params.use_gpu = false; + auto started = Clock::now(); + auto * context = whisper_vad_init_from_file_with_params(argv[1], context_params); + const double load_ms = elapsed(started); + if (!context || failed.load()) { if (context) whisper_vad_free(context); return 3; } + started = Clock::now(); + const bool detected = whisper_vad_detect_speech(context, samples.data(), static_cast(samples.size())); + const double detect_ms = elapsed(started); + const int count = whisper_vad_n_probs(context); + const float * probs = whisper_vad_probs(context); + bool valid = detected && !failed.load() && probs && count == static_cast((samples.size() + 511) / 512); + for (int i = 0; valid && i < count; ++i) valid = std::isfinite(probs[i]) && probs[i] >= 0 && probs[i] <= 1; + if (!valid) { whisper_vad_free(context); return 3; } + started = Clock::now(); + const whisper_vad_params params{0.5f, 250, 100, FLT_MAX, 30, 0.1f}; + auto * segments = whisper_vad_segments_from_probs(context, params); + const double segment_ms = elapsed(started); + if (!segments || failed.load()) { if (segments) whisper_vad_free_segments(segments); whisper_vad_free(context); return 3; } + float previous_end = 0; + for (int i = 0; i < whisper_vad_segments_n_segments(segments); ++i) { + const float start = whisper_vad_segments_get_segment_t0(segments, i); + const float end = whisper_vad_segments_get_segment_t1(segments, i); + if (!std::isfinite(start) || !std::isfinite(end) || start < previous_end || end <= start) { + whisper_vad_free_segments(segments); whisper_vad_free(context); return 3; + } + previous_end = end; + } + // Emit only after all native operations succeed. Raw centiseconds retained; + // the caller maps intervals to verified source bounds and absolute origin. + std::cout << std::setprecision(9) << "{\"schema\":1,\"sample_rate\":16000,\"samples\":" << samples.size() + << ",\"threads\":1,\"gpu\":false,\"load_ms\":" << load_ms + << ",\"detect_ms\":" << detect_ms << ",\"segment_ms\":" << segment_ms << ",\"probabilities\":["; + for (int i = 0; i < count; ++i) { if (i) std::cout << ','; std::cout << probs[i]; } + std::cout << "],\"intervals_cs\":["; + for (int i = 0; i < whisper_vad_segments_n_segments(segments); ++i) { + if (i) std::cout << ','; + std::cout << '[' << whisper_vad_segments_get_segment_t0(segments, i) << ',' + << whisper_vad_segments_get_segment_t1(segments, i) << ']'; + } + std::cout << "]}\n"; + whisper_vad_free_segments(segments); + whisper_vad_free(context); + return std::cout.good() ? 0 : 4; +} diff --git a/tools/speech-vad/test_native.py b/tools/speech-vad/test_native.py new file mode 100644 index 0000000..aae75f1 --- /dev/null +++ b/tools/speech-vad/test_native.py @@ -0,0 +1,53 @@ +"""Actual CPU model contract checks; explicit asset paths required.""" +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import tempfile +import unittest + + +class NativeContract(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.worker = Path(os.environ["SEARCHI_VAD_DIAGNOSTIC"]) + cls.model = Path(os.environ["SEARCHI_VAD_MODEL"]) + if hashlib.sha256(cls.model.read_bytes()).hexdigest() != "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987": + raise ValueError("Expected frozen Silero v6.2.0 export") + + def run_pcm(self, data): + with tempfile.TemporaryDirectory(prefix="searchi-vad-contract-") as scratch: + pcm = Path(scratch) / "samples.f32" + pcm.write_bytes(data) + return subprocess.run([str(self.worker), str(self.model), str(pcm)], + capture_output=True, timeout=30, check=False) + + def test_real_model_silence_tail(self): + result = self.run_pcm(b"\0" * (16001 * 4)) + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", errors="replace")) + report = json.loads(result.stdout) + self.assertEqual(report["samples"], 16001) + self.assertEqual(len(report["probabilities"]), 32) + self.assertEqual(report["intervals_cs"], []) + self.assertEqual(report["threads"], 1) + self.assertIs(report["gpu"], False) + for key in ("load_ms", "detect_ms", "segment_ms"): + self.assertGreaterEqual(report[key], 0) + + def test_invalid_pcm(self): + for data in (b"", b"x", struct.pack("