karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Freeze normal-runtime speech timing regression without timestamp selection

Browse Source
This commit is contained in:
karl committed 2026-10-13 20:46:31 +02:00
1 parent e68f4fcadc
commit 2c3c91179b
7 files changed
+819 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/speech-eval/native-validation/src/main.rs
+171 -31
View File
@@ -15,6 +15,35 @@ use std::{
time::Instant,
};
const TIMING_POLICY_SHA: &str = "816b727f0447f2412b1fbe03a32ad928bdbe9c4f440bc0e7543e4b89c4dad3f0";
const MEDIA_FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
fn timing_sources(fixture: &Value, root: &Path) -> Result<Vec<Value>> {
let mut values = Vec::new();
for id in ["clip04", "music01", "music02", "sound01", "sound02"] {
let matches: Vec<_> = fixture["assets"]
.as_array()
.context("Fixture assets missing")?
.iter()
.filter(|asset| asset["id"] == id)
.collect();
ensure!(
matches.len() == 1,
"Required timing source missing or duplicated"
);
let asset = matches[0];
let path = root.join(asset["path"].as_str().context("Asset path missing")?);
ensure!(
fs::metadata(&path)?.len()
== asset["bytes"].as_u64().context("Asset byte size missing")?
&& file_hash(&path)? == asset["sha256"],
"Timing source bytes changed"
);
values.push(json!({"id":id,"path":path,"sha256":asset["sha256"]}));
}
Ok(values)
}
fn hash(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
@@ -40,14 +69,14 @@ fn write_new(path: &Path, value: &Value) -> Result<()> {
file.sync_all()?;
Ok(())
}
fn windows(start: u32, end: u32) -> Result<Vec<AudioWindow>> {
fn windows(start: u32, end: u32, limit: usize) -> Result<Vec<AudioWindow>> {
ensure!(start <= end, "Invalid source range");
let mut values = Vec::new();
let mut next = start;
while next < end {
ensure!(
values.len() < 4,
"Mini fixture exceeded bounded window count"
values.len() < limit,
"Fixture exceeded bounded window count"
);
values.push(AudioWindow {
start_ms: next,
@@ -59,8 +88,8 @@ fn windows(start: u32, end: u32) -> Result<Vec<AudioWindow>> {
next = next.checked_add(28000).context("Source offset overflow")?;
}
ensure!(
values.len() <= 4,
"Mini fixture exceeded bounded window count"
values.len() <= limit,
"Fixture exceeded bounded window count"
);
Ok(values)
}
@@ -118,6 +147,8 @@ fn run_case(
queries: &[Value],
assets: &Assets,
output: &Path,
max_windows: usize,
point_labels: bool,
) -> Result<Value> {
let case = output.join(name);
fs::create_dir(&case)?;
@@ -133,11 +164,15 @@ fn run_case(
source["sha256"] == file_hash(&input)?,
"Frozen prepared source changed"
);
let path = root.join(format!("{id}.wav"));
let extension = input
.extension()
.and_then(|value| value.to_str())
.context("Original extension missing")?;
let path = root.join(format!("{id}.{extension}"));
fs::copy(&input, &path)?;
let metadata = probe(&assets.tools, &path, &limits)?;
let mut source_windows = Vec::new();
for window in windows(metadata.audio_start_ms, metadata.audio_end_ms)? {
for window in windows(metadata.audio_start_ms, metadata.audio_end_ms, max_windows)? {
let pcm = decode_audio_aligned(&assets.tools, &path, window, &limits)?;
ensure!(
pcm.len() == window.duration_ms as usize * 16,
@@ -241,7 +276,7 @@ fn run_case(
};
hits.push(json!({"asset":id,"result":hit,"verified_recognized_excerpt":excerpt}));
}
let row = json!({"query":query,"hits":hits,"metrics":response.metrics,"strict_point_score_eligible":false});
let row = json!({"query":query,"hits":hits,"metrics":response.metrics,"strict_point_score_eligible":point_labels});
write_new(
&case.join(format!(
"query-{}.json",
@@ -257,12 +292,18 @@ fn run_case(
"Source changed during evaluation"
);
}
let report = json!({"name":name,"sources":source_rows,"lexical_report":discovery,"index_report":indexed,"expected_native_calls":expected_calls,"native_calls":recorded,"published_speech":chunks,"verified_citations":citations,"rows":rows,"native_fingerprint":fingerprint,"google_index_worker":google_index_worker,"google_query_worker":reader.worker_info(),"lexical_ms":lexical_ms,"speech_load_verify_ms":speech_load_ms,"google_index_load_verify_ms":google_index_load_ms,"indexing_ms":indexing_ms,"google_query_load_verify_ms":google_query_load_ms,"query_warmup_ms":warmup_ms,"comparison_eligible":complete_calls&&indexed.speech_failures.is_empty()&&indexed.failures.is_empty(),"strict_point_score_eligible":false});
let report = json!({"name":name,"sources":source_rows,"lexical_report":discovery,"index_report":indexed,"expected_native_calls":expected_calls,"native_calls":recorded,"published_speech":chunks,"verified_citations":citations,"rows":rows,"native_fingerprint":fingerprint,"google_index_worker":google_index_worker,"google_query_worker":reader.worker_info(),"lexical_ms":lexical_ms,"speech_load_verify_ms":speech_load_ms,"google_index_load_verify_ms":google_index_load_ms,"indexing_ms":indexing_ms,"google_query_load_verify_ms":google_query_load_ms,"query_warmup_ms":warmup_ms,"comparison_eligible":complete_calls&&indexed.speech_failures.is_empty()&&indexed.failures.is_empty(),"strict_point_score_eligible":point_labels});
write_new(&case.join("report.json"), &report)?;
Ok(report)
}
fn main() -> Result<()> {
let args: Vec<_> = std::env::args_os().skip(1).collect();
let mut args: Vec<_> = std::env::args_os().skip(1).collect();
let timing = args
.first()
.is_some_and(|value| value == "--media-v1-timing");
if timing {
args.remove(0);
}
ensure!(args.len()==10,"Supply FIXTURE PREPARED GOOGLE_MODELS GOOGLE_WORKER SPEECH_WORKER BASE VAD FFMPEG FFPROBE NEW_OUTPUT_DIRECTORY");
ensure!(
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
@@ -271,16 +312,30 @@ fn main() -> Result<()> {
let fixture_path = PathBuf::from(&args[0]);
let fixture_raw = fs::read(&fixture_path)?;
let fixture: Value = serde_json::from_slice(&fixture_raw)?;
let input_lock: Value =
serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
let prepared_raw = fs::read(&args[1])?;
let prepared: Value = serde_json::from_slice(&prepared_raw)?;
ensure!(
input_lock["fixture_sha256"] == hash(&fixture_raw)
&& input_lock["prepared_sha256"] == hash(&prepared_raw)
&& prepared["fixture_sha256"] == hash(&fixture_raw),
"Frozen fixture/prepared plan differs"
);
let prepared: Value = if timing {
ensure!(
hash(&fixture_raw) == MEDIA_FIXTURE_SHA && hash(&prepared_raw) == TIMING_POLICY_SHA,
"Frozen media-v1 timing policy or labels changed"
);
let policy: Value = serde_json::from_slice(&prepared_raw)?;
let clips = timing_sources(
&fixture,
fixture_path.parent().context("Fixture root absent")?,
)?;
json!({"clips":clips,"policy":policy})
} else {
let input_lock: Value =
serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
let value: Value = serde_json::from_slice(&prepared_raw)?;
ensure!(
input_lock["fixture_sha256"] == hash(&fixture_raw)
&& input_lock["prepared_sha256"] == hash(&prepared_raw)
&& value["fixture_sha256"] == hash(&fixture_raw),
"Frozen fixture/prepared plan differs"
);
value
};
let assets = Assets {
model: PathBuf::from(&args[2]),
google: PathBuf::from(&args[3]),
@@ -289,11 +344,32 @@ fn main() -> Result<()> {
vad: PathBuf::from(&args[6]),
tools: FfmpegTools::new(PathBuf::from(&args[7]), PathBuf::from(&args[8]))?,
};
ensure!(
prepared["ffmpeg_sha256"] == file_hash(&assets.tools.ffmpeg)?
&& prepared["ffprobe_sha256"] == file_hash(&assets.tools.ffprobe)?,
"Frozen decoder differs"
);
if timing {
ensure!(
prepared["policy"]["speech_worker_sha256"] == file_hash(&assets.speech)?
&& prepared["policy"]["speech_inventory_sha256"]
== file_hash(
&assets
.speech
.parent()
.context("Runtime root")?
.join("runtime-manifest.json")
)?,
"Frozen normal runtime changed"
);
ensure!(
prepared["policy"]["google_worker_sha256"] == file_hash(&assets.google)?
&& prepared["policy"]["ffmpeg_sha256"] == file_hash(&assets.tools.ffmpeg)?
&& prepared["policy"]["ffprobe_sha256"] == file_hash(&assets.tools.ffprobe)?,
"Frozen Google worker or decoder changed"
);
} else {
ensure!(
prepared["ffmpeg_sha256"] == file_hash(&assets.tools.ffmpeg)?
&& prepared["ffprobe_sha256"] == file_hash(&assets.tools.ffprobe)?,
"Frozen decoder differs"
);
}
let output = PathBuf::from(&args[9]);
fs::create_dir(&output)?;
let mut provenance = BTreeMap::new();
@@ -308,15 +384,51 @@ fn main() -> Result<()> {
] {
provenance.insert(file, file_hash(Path::new(file))?);
}
let source_clock_policy = if timing {
"ActualService speech30s overlap2s; independentPCM window hashes; original frozen publisher timed labels"
} else {
"ActualService speech30s overlap2s; independently decodedPCM window hashes; no strict word labels"
};
write_new(
&output.join("preflight.json"),
&json!({"fixture_sha256":hash(&fixture_raw),"prepared_sha256":hash(&prepared_raw),"binary_sha256":file_hash(&std::env::current_exe()?)?,"speech_worker_sha256":file_hash(&assets.speech)?,"speech_inventory_sha256":file_hash(&assets.speech.parent().context("Runtime directory")?.join("runtime-manifest.json"))?,"base_sha256":file_hash(&assets.base)?,"vad_sha256":file_hash(&assets.vad)?,"google_worker_sha256":file_hash(&assets.google)?,"source_hashes_at_start":provenance,"speech_policy":searchi_speech::POLICY,"source_clock_policy":"ActualService speech30s overlap2s; independently decodedPCM window hashes; no strict word labels","force_cpu":true,"threads":1}),
&json!({"fixture_sha256":hash(&fixture_raw),"prepared_sha256":hash(&prepared_raw),"binary_sha256":file_hash(&std::env::current_exe()?)?,"speech_worker_sha256":file_hash(&assets.speech)?,"speech_inventory_sha256":file_hash(&assets.speech.parent().context("Runtime directory")?.join("runtime-manifest.json"))?,"base_sha256":file_hash(&assets.base)?,"vad_sha256":file_hash(&assets.vad)?,"google_worker_sha256":file_hash(&assets.google)?,"ffmpeg_sha256":file_hash(&assets.tools.ffmpeg)?,"ffprobe_sha256":file_hash(&assets.tools.ffprobe)?,"source_hashes_at_start":provenance,"speech_policy":searchi_speech::POLICY,"source_clock_policy":source_clock_policy,"force_cpu":true,"threads":1}),
)?;
if timing {
write_new(&output.join("timing-policy.json"), &prepared["policy"])?;
let sources = prepared["clips"]
.as_array()
.context("Five timing sources absent")?;
let queries: Vec<_> = fixture["queries"]
.as_array()
.context("Queries absent")?
.iter()
.filter(|query| query["category"] == "video-speech")
.cloned()
.collect();
ensure!(
queries.len() == 8,
"All eight frozen timing queries required"
);
let result = match run_case("timing", sources, &queries, &assets, &output, 12, true) {
Ok(value) => value,
Err(error) => {
write_new(
&output.join("failure.json"),
&json!({"complete":false,"error":format!("{error:#}"),"comparison_eligible":false}),
)?;
return Err(error);
}
};
return write_new(
&output.join("report.json"),
&json!({"complete":true,"fixture_sha256":hash(&fixture_raw),"timing_policy_sha256":TIMING_POLICY_SHA,"timing":result,"strict_point_score_eligible":true,"production_policy_changed":false,"scope":"Actual shared service hybrid first hits, same completefive-source timingpool; normalauto speech runtime forcedCPU; live30s/2soverlap; no label selected spans or carried-over score"}),
);
}
let core = prepared["clips"].as_array().context("Core clips missing")?;
ensure!(core.len() == 4, "All four controls required");
let queries = fixture["queries"].as_array().context("Queries missing")?;
ensure!(queries.len() == 10, "All ten queries required");
let first = match run_case("core", core, queries, &assets, &output) {
let first = match run_case("core", core, queries, &assets, &output, 4, false) {
Ok(value) => value,
Err(error) => {
write_new(
@@ -330,7 +442,7 @@ fn main() -> Result<()> {
.as_array()
.context("Seam controls missing")?;
ensure!(seams.len() == 2, "Both seam sources required");
let second = match run_case("seams", seams, &[], &assets, &output) {
let second = match run_case("seams", seams, &[], &assets, &output, 4, false) {
Ok(value) => value,
Err(error) => {
write_new(
@@ -350,21 +462,49 @@ mod tests {
use super::*;
#[test]
fn overlap_maps_complete_seam_tail() {
let value = windows(0, 40024).unwrap();
let value = windows(0, 40024, 4).unwrap();
assert_eq!(value.len(), 2);
assert_eq!(value[1].start_ms, 28000);
assert_eq!(value[1].duration_ms, 12024);
}
#[test]
fn positioned_short_source_keeps_origin() {
let value = windows(2000, 4000).unwrap();
let value = windows(2000, 4000, 4).unwrap();
assert_eq!(value[0].start_ms, 2000);
assert_eq!(value[0].duration_ms, 2000);
}
#[test]
fn invalid_or_large_fixture_ranges_fail() {
assert!(windows(3, 2).is_err());
assert!(windows(0, 200000).is_err());
assert!(windows(3, 2, 4).is_err());
assert!(windows(0, 200000, 4).is_err());
}
#[test]
fn complete_piano_timing_windows_remain_bounded() {
let values = windows(0, 254454, 12).unwrap();
assert_eq!(values.len(), 10);
assert_eq!(values.last().unwrap().start_ms, 252000);
assert_eq!(values.last().unwrap().duration_ms, 2454);
assert!(values.iter().all(|window| window.duration_ms <= 30000));
}
#[test]
#[ignore = "Requires the five hash-verified real media fixture files; no inference"]
fn timing_mode_preflight_checks_all_real_frozen_sources_without_models() -> Result<()> {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../fixtures/media");
let raw = fs::read(root.join("fixture.json"))?;
ensure!(hash(&raw) == MEDIA_FIXTURE_SHA, "Frozen fixture changed");
let fixture: Value = serde_json::from_slice(&raw)?;
let values = timing_sources(&fixture, &root)?;
assert_eq!(values.len(), 5);
assert!(values
.iter()
.any(|source| Path::new(source["path"].as_str().unwrap())
.extension()
.is_some_and(|value| value == "mp4")));
ensure!(
hash(include_bytes!("../media-timing-policy.json")) == TIMING_POLICY_SHA,
"Frozen timing policy changed"
);
Ok(())
}
#[test]
fn existing_raw_output_is_preserved() -> Result<()> {