diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 3843beb..aa31bd5 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -82,6 +82,9 @@ release. Keep their evidence and unchecked production gates intact. - [x] Build a pinned CPU VAD diagnostic with explicit one-thread inference, native error detection and full probabilities; actual Silero silence/input contracts passed. Media quality comparison remains open (see SPEECH-VAD.md). + - [x] Complete frozen five-source VAD/base-ASR/Google CPU diagnostic: + component costs decreased, but localization stayed 4/8 point and 6/8 overlap; + no production default selected. Independent held-out media quality remains open. - [ ] Run genuine local speech candidates, Google transcript retrieval and held-out English/German/music/no-speech checks before selecting a default. - [ ] Package and verify decoders on every release OS; test production acceleration. diff --git a/docs/SPEECH-VAD.md b/docs/SPEECH-VAD.md index 948260b..260d17e 100644 --- a/docs/SPEECH-VAD.md +++ b/docs/SPEECH-VAD.md @@ -243,11 +243,44 @@ segments versus nineteen; segment count does not prove transcript accuracy. Music/sound ASR text vanished through explicit no-speech decisions, not manual phrase filtering. Original Google audio retrieval remains independent. -The next gate is genuine Google retrieval on all eight unchanged queries and -independent speech recall/alignment assessment. Do not select a production +Genuine Google retrieval on all eight unchanged queries is now measured below; +independent speech recall/alignment assessment remains open. Do not select a production policy from timing or this tiny corpus alone. A future persistent ASR session could reduce model-start overhead but must be measured independently. +## Genuine Google comparison result + +The updated Rust diagnostic verifies the exact embedded frozen VAD bytes, +reconstructs every source interval window and requires explicit no-speech status +for the four empty competitors. The original ungated coverage contract remains +strict. Six Rust tests passed, including omission/tampering rejection; the +agent also ran strict Clippy and built the release executable. Its recorded +SHA-256 matches the current executable: +`5a8ecb9e2ff571610cfe9f0dcd4876953d8975ff4a84972d321cd91636ceaf84`. + +Actual Google EmbeddingGemma 2 CPU document/query inference completed using the +normal auto-capable worker with CPU forced. All twenty recognized segments +entered retrieval without relevance-label filtering; all five original source +identities remained verified. The complete, eligible ranking report is ignored +`.searchi/speech/gated-base-google-ranks-v2.json`, SHA-256 +`1647cf5826dbeaf43f8c47209ca40af6bc2b68cd56f0c5f44eda36ed2eab8a30`. +The scorer report SHA-256 is +`3d15e72764cf2329b31caf9020af9182a91acb7229b13d7dec1b24464351c0dc`. +An initial run used a nonexistent worker path and retained its incomplete +verification checkpoint; it performed no successful inference and was not scored. + +| Fixed eight-query metric | Ungated base | VAD-gated base | +| --- | ---: | ---: | +| Correct top file | 8/8 | 8/8 | +| Top start point within labelled interval | 4/8 | 4/8 | +| Top interval overlaps labelled interval | 6/8 | 6/8 | + +The speech gate removed incidental music/sound transcripts and reduced this +pool's processing cost, but did **not** improve these localization metrics. +It is not selected as a production default. More independent speech coverage, +recognition and alignment work is needed before adding transcription enrichment. +The original Google media embeddings remain the product's media search path. + Open gates: actual one-thread audit, detection/recognition comparison, ordinary and DTW timestamp regression, independent held-out quality, background starvation/cancellation/crash recovery, packaging/license review, diff --git a/tools/speech-rank/README.md b/tools/speech-rank/README.md index ccd6b6e..06eaae7 100644 --- a/tools/speech-rank/README.md +++ b/tools/speech-rank/README.md @@ -19,6 +19,17 @@ ranks are retained; at most the first 100 segment hits per query are exported fo the existing scorer. Ties use stable source order. No confidence threshold or oracle matching is applied. +The additional frozen `gated-vad-v1` diagnostic policy accepts only the exact +VAD report SHA-256 `093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df`. +The ASR report embeds its original UTF-8 bytes and parsed document; both must +agree. Source probes, decoder hashes and the pinned base Whisper worker must +match. The ranker independently rebuilds every contiguous window from the frozen +bounded source-sample intersections, retaining interval identity and explicit +sub-millisecond tail remainder. Omitted intervals, changed windows or claimed +no-speech status without frozen empty intervals are rejected before inference. +All five sources remain explicit competitors; no relevance label selects which +source receives ASR. Existing ungated reports still require full audio coverage. + ```powershell cargo test --manifest-path tools/speech-rank/Cargo.toml --locked cargo build --release --manifest-path tools/speech-rank/Cargo.toml --locked diff --git a/tools/speech-rank/src/gated.rs b/tools/speech-rank/src/gated.rs new file mode 100644 index 0000000..4efe310 --- /dev/null +++ b/tools/speech-rank/src/gated.rs @@ -0,0 +1,193 @@ +//! Gated diagnostics are anchored to exact frozen VAD bytes, never claimed windows. +use super::*; +pub const POLICY: &str = "gated-vad-v1;source-intersections;integer-ms-tail-remainder-recorded;contiguous30s;independent-process;auto-language;segment-times"; +const VAD_SHA: &str = "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df"; + +pub fn verified_report(asr: &Value) -> Result> { + if asr["policy"] != POLICY { + return Ok(None); + } + let bytes = asr["vad_report_utf8"] + .as_str() + .context("Missing original VAD bytes")? + .as_bytes(); + ensure!( + bytes.len() <= 16 * 1024 * 1024 + && format!("{:x}", Sha256::digest(bytes)) == VAD_SHA + && asr["vad_report_sha256"] == VAD_SHA, + "Frozen VAD bytes changed" + ); + let vad: Value = serde_json::from_slice(bytes)?; + ensure!( + asr["actual_vad_report"] == vad, + "VAD parsed provenance differs from original bytes" + ); + ensure!( + asr["candidate"] == "base" + && asr["worker_sha256"] + == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8", + "Unknown gated ASR worker/model" + ); + for name in ["ffmpeg_sha256", "ffprobe_sha256"] { + ensure!(asr[name] == vad[name], "VAD and ASR decoder differ"); + } + Ok(Some(vad)) +} + +pub fn planned_windows(vad: &Value, row: &Value, asset: &Value) -> Result> { + let detected = vad["rows"] + .as_array() + .context("Missing frozen VAD rows")? + .iter() + .find(|v| v["asset"] == row["asset"]) + .context("Missing VAD competitor")?; + ensure!( + detected["source_sha256"] == asset["sha256"] && row["actual_probe"] == detected["probe"], + "VAD source/probe differs" + ); + let audio = detected["probe"]["streams"] + .as_array() + .context("Missing audio probe")? + .iter() + .find(|v| v["codec_type"] == "audio") + .context("Missing primary audio")?; + let start: f64 = audio["start_time"].as_str().unwrap_or("0").parse()?; + let duration: f64 = audio["duration"] + .as_str() + .context("Missing audio duration")? + .parse()?; + ensure!( + start.is_finite() + && start >= 0.0 + && duration.is_finite() + && duration > 0.0 + && duration <= 600.0, + "Invalid source timeline" + ); + let origin = (start * 1000.0).ceil() as u64; + let end = ((start + duration) * 1000.0).ceil() as u64; + ensure!( + row["audio_start_ms"] == origin + && row["audio_end_ms"] == end + && row["audio_stream_index"] == audio["index"], + "ASR timeline differs from frozen probe" + ); + let samples = number(&detected["native"]["samples"])?; + let intervals = detected["intervals"] + .as_array() + .context("Missing VAD intervals")?; + let mut result = Vec::new(); + let mut previous = 0; + for (index, interval) in intervals.iter().enumerate() { + let raw_start = number(&interval["raw_cs"][0])? + .checked_mul(160) + .context("Interval overflow")?; + let raw_end = number(&interval["raw_cs"][1])? + .checked_mul(160) + .context("Interval overflow")?; + let first = raw_start.min(samples); + let last = raw_end.min(samples); + ensure!( + raw_end > raw_start + && first >= previous + && interval["source_samples"] == json!([first, last]) + && interval["tail_padding_samples"] == raw_end.saturating_sub(samples), + "Invalid bounded VAD intersection" + ); + previous = last; + let mut offset = origin + first / 16; + let end = origin + last / 16; + while offset < end { + let duration = (end - offset).min(30000); + result.push(json!({"offset_ms":offset,"duration_ms":duration,"vad_interval_index":index,"vad_source_intersection":interval,"unrepresented_tail_samples":last%16})); + ensure!(result.len() <= 20, "Too many gated windows"); + offset += duration; + } + } + ensure!( + row["status"] + == if result.is_empty() { + "no_speech" + } else { + "speech" + } + && row["asr_not_run"] == result.is_empty(), + "Missing explicit speech/no-speech evidence" + ); + Ok(result) +} + +pub fn matches_plan(window: &Value, planned: &Value) -> bool { + [ + "offset_ms", + "duration_ms", + "vad_interval_index", + "vad_source_intersection", + "unrepresented_tail_samples", + ] + .iter() + .all(|key| window[key] == planned[key]) +} + +pub fn verify_windows(windows: &[Value], planned: &[Value]) -> Result<()> { + ensure!( + windows.len() == planned.len() + && windows + .iter() + .zip(planned) + .all(|(window, plan)| matches_plan(window, plan)), + "Incomplete or changed VAD interval windows" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn source(intervals: Value) -> (Value, Value, Value) { + let probe = json!({"streams":[{"codec_type":"audio","index":2,"start_time":"0.125","duration":"65.0"}]}); + let vad = json!({"rows":[{"asset":"clip04","source_sha256":"sha","probe":probe,"native":{"samples":1040000},"intervals":intervals}]}); + let row = json!({"asset":"clip04","actual_probe":probe,"audio_start_ms":125,"audio_end_ms":65125,"audio_stream_index":2,"status":"speech","asr_not_run":false}); + (vad, row, json!({"sha256":"sha"})) + } + #[test] + fn windows_reconstruct_original_intervals_without_merging() { + let intervals = json!([ + {"raw_cs":[0,3100],"source_samples":[0,496000],"tail_padding_samples":0}, + {"raw_cs":[4000,4200],"source_samples":[640000,672000],"tail_padding_samples":0} + ]); + let (vad, row, asset) = source(intervals); + let plan = planned_windows(&vad, &row, &asset).unwrap(); + assert_eq!(plan.len(), 3); + assert_eq!(plan[0]["offset_ms"], 125); + assert_eq!(plan[1]["offset_ms"], 30125); + assert_eq!(plan[1]["duration_ms"], 1000); + assert_eq!(plan[2]["offset_ms"], 40125); + assert!(verify_windows(&plan, &plan).is_ok()); + assert!(verify_windows(&plan[..2], &plan).is_err()); + let mut omitted = plan[2].clone(); + omitted["vad_interval_index"] = json!(0); + assert!(!matches_plan(&omitted, &plan[2])); + let mut changed = vad.clone(); + changed["rows"][0]["intervals"][0]["source_samples"][1] = json!(480000); + assert!(planned_windows(&changed, &row, &asset).is_err()); + } + #[test] + fn no_speech_requires_empty_frozen_intervals_and_explicit_status() { + let (vad, mut row, asset) = source(json!([])); + assert!(planned_windows(&vad, &row, &asset).is_err()); + row["status"] = json!("no_speech"); + row["asr_not_run"] = json!(true); + assert!(planned_windows(&vad, &row, &asset).unwrap().is_empty()); + row["audio_start_ms"] = json!(0); + assert!(planned_windows(&vad, &row, &asset).is_err()); + } + #[test] + fn arbitrary_vad_claim_cannot_authorize_candidate_exclusion() { + let report = json!({"policy":POLICY,"vad_report_sha256":VAD_SHA,"vad_report_utf8":"{}","actual_vad_report":{}}); + assert!(verified_report(&report).is_err()); + assert!(verified_report(&json!({"policy":super::super::POLICY})) + .unwrap() + .is_none()); + } +} diff --git a/tools/speech-rank/src/main.rs b/tools/speech-rank/src/main.rs index 54909e9..eb3577c 100644 --- a/tools/speech-rank/src/main.rs +++ b/tools/speech-rank/src/main.rs @@ -13,6 +13,7 @@ use std::{ const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393"; const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"]; +mod gated; const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times"; fn digest(path: &Path) -> Result { @@ -90,7 +91,7 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { ensure!( asr["complete"] == true && asr["fixture_sha256"] == FIXTURE_SHA - && asr["policy"] == POLICY + && matches!(asr["policy"].as_str(), Some(POLICY) | Some(gated::POLICY)) && asr["backend"] == "forced CPU" && asr["threads"] == 1, "Incomplete, stale, or unrecognized ASR report" @@ -104,6 +105,7 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { ] { ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance"); } + let vad = gated::verified_report(asr)?; let rows = asr["rows"].as_array().context("Missing ASR rows")?; ensure!(rows.len() == 5, "All five competing assets required"); let mut seen = BTreeSet::new(); @@ -133,12 +135,23 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { ); let windows = row["windows"].as_array().context("Missing windows")?; ensure!(windows.len() <= 20, "Oversized window list"); + let planned = vad + .as_ref() + .map(|vad| gated::planned_windows(vad, row, asset)) + .transpose()?; + if let Some(planned) = &planned { + gated::verify_windows(windows, planned)?; + } let mut next = start; for (window_index, window) in windows.iter().enumerate() { let offset = number(&window["offset_ms"])?; let duration = number(&window["duration_ms"])?; ensure!( - offset == next && duration == (end - offset).min(30000) && duration > 0, + if let Some(planned) = &planned { + gated::matches_plan(window, &planned[window_index]) + } else { + offset == next && duration == (end - offset).min(30000) && duration > 0 + }, "Missing, overlapping or truncated source windows" ); next = offset.checked_add(duration).context("Window overflow")?; @@ -183,7 +196,10 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { ensure!(segments.len() <= 5000, "Too many recognized segments"); } } - ensure!(next == end, "Incomplete source-window coverage"); + ensure!( + planned.is_some() || next == end, + "Incomplete source-window coverage" + ); } ensure!( asr["quality_failed"] == !excluded.is_empty(), @@ -197,6 +213,12 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { == excluded.len(), "ASR validation failure count inconsistent" ); + if vad.is_some() { + ensure!( + asr["comparison_eligible"] == excluded.is_empty(), + "Gated comparison status inconsistent" + ); + } Ok((segments, excluded)) }