379 lines
16 KiB
Rust
379 lines
16 KiB
Rust
//! Frozen actual shared-service speech validation; no reference text enters inference.
|
|
use anyhow::{ensure, Context, Result};
|
|
use searchi_core::SearchIndex;
|
|
use searchi_media::{decode_audio_aligned, probe, AudioWindow, FfmpegTools, MediaLimits};
|
|
use searchi_service::{HybridSearch, SearchOptions, SpeechProvider, SpeechTranscription};
|
|
use searchi_speech::{Device, SpeechWorker};
|
|
use serde_json::{json, Value};
|
|
use sha2::{Digest, Sha256};
|
|
use std::{
|
|
collections::BTreeMap,
|
|
fs,
|
|
io::{Read, Write},
|
|
path::{Path, PathBuf},
|
|
sync::{Arc, Mutex},
|
|
time::Instant,
|
|
};
|
|
|
|
fn hash(bytes: &[u8]) -> String {
|
|
format!("{:x}", Sha256::digest(bytes))
|
|
}
|
|
fn file_hash(path: &Path) -> Result<String> {
|
|
let mut file = fs::File::open(path)?;
|
|
let mut state = Sha256::new();
|
|
let mut buffer = [0; 65536];
|
|
loop {
|
|
let n = file.read(&mut buffer)?;
|
|
if n == 0 {
|
|
break;
|
|
}
|
|
state.update(&buffer[..n]);
|
|
}
|
|
Ok(format!("{:x}", state.finalize()))
|
|
}
|
|
fn write_new(path: &Path, value: &Value) -> Result<()> {
|
|
let mut file = fs::OpenOptions::new()
|
|
.write(true)
|
|
.create_new(true)
|
|
.open(path)?;
|
|
file.write_all(&serde_json::to_vec_pretty(value)?)?;
|
|
file.sync_all()?;
|
|
Ok(())
|
|
}
|
|
fn windows(start: u32, end: u32) -> Result<Vec<AudioWindow>> {
|
|
ensure!(start <= end, "Invalid source range");
|
|
let mut values = Vec::new();
|
|
let mut next = start;
|
|
while next < end {
|
|
ensure!(
|
|
values.len() < 4,
|
|
"Mini fixture exceeded bounded window count"
|
|
);
|
|
values.push(AudioWindow {
|
|
start_ms: next,
|
|
duration_ms: (end - next).min(30000),
|
|
});
|
|
if end - next <= 30000 {
|
|
break;
|
|
}
|
|
next = next.checked_add(28000).context("Source offset overflow")?;
|
|
}
|
|
ensure!(
|
|
values.len() <= 4,
|
|
"Mini fixture exceeded bounded window count"
|
|
);
|
|
Ok(values)
|
|
}
|
|
|
|
struct Recorder {
|
|
worker: SpeechWorker,
|
|
expected: BTreeMap<String, Vec<Value>>,
|
|
calls: Arc<Mutex<Vec<Value>>>,
|
|
directory: PathBuf,
|
|
}
|
|
impl SpeechProvider for Recorder {
|
|
fn fingerprint(&self) -> &str {
|
|
self.worker.fingerprint()
|
|
}
|
|
fn transcribe(&mut self, pcm: &[f32], rate: u32) -> Result<SpeechTranscription> {
|
|
let pcm_bytes: Vec<_> = pcm.iter().flat_map(|value| value.to_le_bytes()).collect();
|
|
let sha = hash(&pcm_bytes);
|
|
let expected = self
|
|
.expected
|
|
.get(&sha)
|
|
.context("Service supplied PCM not in independently verified source windows")?;
|
|
let started = Instant::now();
|
|
let result = self.worker.transcribe(pcm, rate);
|
|
let mut record = json!({"pcm_sha256":sha,"samples":pcm.len(),"sample_rate":rate,"expected_source_windows":expected,"native_fingerprint":self.worker.fingerprint(),"wall_ms":started.elapsed().as_secs_f64()*1000.0,"scope":"Exact accepted SpeechTranscription or returned error; native wire bytes/stderr unavailable through public client API"});
|
|
match &result {
|
|
Ok(value) => record["transcription"] = serde_json::to_value(value)?,
|
|
Err(error) => record["error"] = json!(format!("{error:#}")),
|
|
}
|
|
let mut calls = self
|
|
.calls
|
|
.lock()
|
|
.map_err(|_| anyhow::anyhow!("Recording mutex poisoned"))?;
|
|
write_new(
|
|
&self
|
|
.directory
|
|
.join(format!("native-call-{:03}.json", calls.len() + 1)),
|
|
&record,
|
|
)?;
|
|
calls.push(record);
|
|
result
|
|
}
|
|
}
|
|
struct Assets {
|
|
model: PathBuf,
|
|
google: PathBuf,
|
|
speech: PathBuf,
|
|
base: PathBuf,
|
|
vad: PathBuf,
|
|
tools: FfmpegTools,
|
|
}
|
|
|
|
fn run_case(
|
|
name: &str,
|
|
sources: &[Value],
|
|
queries: &[Value],
|
|
assets: &Assets,
|
|
output: &Path,
|
|
) -> Result<Value> {
|
|
let case = output.join(name);
|
|
fs::create_dir(&case)?;
|
|
let root = case.join("corpus");
|
|
fs::create_dir(&root)?;
|
|
let mut expected = BTreeMap::<String, Vec<Value>>::new();
|
|
let limits = MediaLimits::default();
|
|
let mut source_rows = Vec::new();
|
|
for source in sources {
|
|
let id = source["id"].as_str().context("ID missing")?;
|
|
let input = PathBuf::from(source["path"].as_str().context("Source path missing")?);
|
|
ensure!(
|
|
source["sha256"] == file_hash(&input)?,
|
|
"Frozen prepared source changed"
|
|
);
|
|
let path = root.join(format!("{id}.wav"));
|
|
fs::copy(&input, &path)?;
|
|
let metadata = probe(&assets.tools, &path, &limits)?;
|
|
let mut source_windows = Vec::new();
|
|
for window in windows(metadata.audio_start_ms, metadata.audio_end_ms)? {
|
|
let pcm = decode_audio_aligned(&assets.tools, &path, window, &limits)?;
|
|
ensure!(
|
|
pcm.len() == window.duration_ms as usize * 16,
|
|
"Fixture source clock coverage differs"
|
|
);
|
|
let sha = hash(
|
|
&pcm.iter()
|
|
.flat_map(|value| value.to_le_bytes())
|
|
.collect::<Vec<_>>(),
|
|
);
|
|
let record = json!({"asset":id,"start_ms":window.start_ms,"duration_ms":window.duration_ms,"samples":pcm.len(),"pcm_sha256":sha});
|
|
expected.entry(sha).or_default().push(record.clone());
|
|
source_windows.push(record);
|
|
}
|
|
source_rows.push(json!({"id":id,"path":path,"sha256":source["sha256"],"windows":source_windows,"audio_start_ms":metadata.audio_start_ms,"audio_end_ms":metadata.audio_end_ms,"piecewise_map":source.get("piecewise_map")}));
|
|
}
|
|
let db = case.join("index.sqlite");
|
|
let started = Instant::now();
|
|
let mut lexical = SearchIndex::open(&db)?;
|
|
let discovery = lexical.index_folder(&root)?;
|
|
let lexical_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
drop(lexical);
|
|
let started = Instant::now();
|
|
let native = SpeechWorker::load(&assets.speech, &assets.base, &assets.vad, Device::Cpu)?;
|
|
let speech_load_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
let fingerprint = native.fingerprint().to_owned();
|
|
let calls = Arc::new(Mutex::new(Vec::new()));
|
|
let recorder = Recorder {
|
|
worker: native,
|
|
expected: expected.clone(),
|
|
calls: calls.clone(),
|
|
directory: case.clone(),
|
|
};
|
|
let started = Instant::now();
|
|
let mut indexer = HybridSearch::open(&db, &assets.model, &assets.google, 1)?
|
|
.with_speech_provider(Box::new(recorder))?;
|
|
let google_index_load_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
let started = Instant::now();
|
|
let indexed = indexer.index_embeddings_with_media(&assets.tools, &limits)?;
|
|
let indexing_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
let google_index_worker = serde_json::to_value(indexer.worker_info())?;
|
|
write_new(
|
|
&case.join("indexed.json"),
|
|
&json!({"sources":source_rows,"index_report":indexed,"fingerprint":fingerprint,"worker":google_index_worker,"indexing_ms":indexing_ms}),
|
|
)?;
|
|
drop(indexer);
|
|
let recorded = calls
|
|
.lock()
|
|
.map_err(|_| anyhow::anyhow!("Recording mutex poisoned"))?
|
|
.clone();
|
|
let expected_calls = expected.values().map(Vec::len).sum::<usize>();
|
|
let complete_calls =
|
|
recorded.len() == expected_calls && recorded.iter().all(|row| row.get("error").is_none());
|
|
let connection =
|
|
rusqlite::Connection::open_with_flags(&db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
|
let mut statement=connection.prepare("SELECT path,number,speech_end_ms,snippet,content_hash FROM semantic_chunks WHERE generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1) AND source_type='speech' ORDER BY path,ordinal")?;
|
|
let chunks=statement.query_map([],|row|Ok(json!({"path":row.get::<_,String>(0)?,"start_ms":row.get::<_,u32>(1)?,"end_ms":row.get::<_,u32>(2)?,"text":row.get::<_,String>(3)?,"content_hash":row.get::<_,String>(4)?})))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
|
let started = Instant::now();
|
|
let mut reader = HybridSearch::open_query(&db, &assets.model, &assets.google, 1)?;
|
|
let google_query_load_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
let mut citations = Vec::new();
|
|
for chunk in &chunks {
|
|
let excerpt = reader.speech_excerpt_matching(
|
|
chunk["path"].as_str().unwrap(),
|
|
chunk["start_ms"].as_u64().unwrap() as u32,
|
|
chunk["text"].as_str().unwrap(),
|
|
4096,
|
|
)?;
|
|
ensure!(
|
|
excerpt
|
|
.as_ref()
|
|
.is_some_and(|value| value.text == chunk["text"].as_str().unwrap()
|
|
&& value.provenance == "local_speech_recognition"),
|
|
"Published recognized citation missing"
|
|
);
|
|
citations.push(serde_json::to_value(excerpt)?);
|
|
}
|
|
let started = Instant::now();
|
|
reader.warmup()?;
|
|
let warmup_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
let mut rows = Vec::new();
|
|
for query in queries {
|
|
let response = reader.search_measured(
|
|
query["text"].as_str().context("Query text missing")?,
|
|
20,
|
|
&SearchOptions::default(),
|
|
true,
|
|
)?;
|
|
let mut hits = Vec::new();
|
|
for hit in response.results {
|
|
let id = Path::new(&hit.filename)
|
|
.file_stem()
|
|
.context("Result ID missing")?
|
|
.to_string_lossy()
|
|
.into_owned();
|
|
let excerpt = if hit.match_kind == "semantic_speech" {
|
|
let location = hit.source.as_ref().context("Speech location missing")?;
|
|
reader.speech_excerpt_matching(&hit.path, location.number, &hit.snippet, 4096)?
|
|
} else {
|
|
None
|
|
};
|
|
hits.push(json!({"asset":id,"result":hit,"verified_recognized_excerpt":excerpt}));
|
|
}
|
|
let row = json!({"query":query,"hits":hits,"metrics":response.metrics,"strict_point_score_eligible":false});
|
|
write_new(
|
|
&case.join(format!(
|
|
"query-{}.json",
|
|
query["id"].as_str().context("Query ID missing")?
|
|
)),
|
|
&row,
|
|
)?;
|
|
rows.push(row);
|
|
}
|
|
for source in &source_rows {
|
|
ensure!(
|
|
source["sha256"] == file_hash(Path::new(source["path"].as_str().unwrap()))?,
|
|
"Source changed during evaluation"
|
|
);
|
|
}
|
|
let report = json!({"name":name,"sources":source_rows,"lexical_report":discovery,"index_report":indexed,"expected_native_calls":expected_calls,"native_calls":recorded,"published_speech":chunks,"verified_citations":citations,"rows":rows,"native_fingerprint":fingerprint,"google_index_worker":google_index_worker,"google_query_worker":reader.worker_info(),"lexical_ms":lexical_ms,"speech_load_verify_ms":speech_load_ms,"google_index_load_verify_ms":google_index_load_ms,"indexing_ms":indexing_ms,"google_query_load_verify_ms":google_query_load_ms,"query_warmup_ms":warmup_ms,"comparison_eligible":complete_calls&&indexed.speech_failures.is_empty()&&indexed.failures.is_empty(),"strict_point_score_eligible":false});
|
|
write_new(&case.join("report.json"), &report)?;
|
|
Ok(report)
|
|
}
|
|
fn main() -> Result<()> {
|
|
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
|
ensure!(args.len()==10,"Supply FIXTURE PREPARED GOOGLE_MODELS GOOGLE_WORKER SPEECH_WORKER BASE VAD FFMPEG FFPROBE NEW_OUTPUT_DIRECTORY");
|
|
ensure!(
|
|
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
|
"Windows VM diagnostic requires CPU"
|
|
);
|
|
let fixture_path = PathBuf::from(&args[0]);
|
|
let fixture_raw = fs::read(&fixture_path)?;
|
|
let fixture: Value = serde_json::from_slice(&fixture_raw)?;
|
|
let input_lock: Value =
|
|
serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
|
|
let prepared_raw = fs::read(&args[1])?;
|
|
let prepared: Value = serde_json::from_slice(&prepared_raw)?;
|
|
ensure!(
|
|
input_lock["fixture_sha256"] == hash(&fixture_raw)
|
|
&& input_lock["prepared_sha256"] == hash(&prepared_raw)
|
|
&& prepared["fixture_sha256"] == hash(&fixture_raw),
|
|
"Frozen fixture/prepared plan differs"
|
|
);
|
|
let assets = Assets {
|
|
model: PathBuf::from(&args[2]),
|
|
google: PathBuf::from(&args[3]),
|
|
speech: PathBuf::from(&args[4]),
|
|
base: PathBuf::from(&args[5]),
|
|
vad: PathBuf::from(&args[6]),
|
|
tools: FfmpegTools::new(PathBuf::from(&args[7]), PathBuf::from(&args[8]))?,
|
|
};
|
|
ensure!(
|
|
prepared["ffmpeg_sha256"] == file_hash(&assets.tools.ffmpeg)?
|
|
&& prepared["ffprobe_sha256"] == file_hash(&assets.tools.ffprobe)?,
|
|
"Frozen decoder differs"
|
|
);
|
|
let output = PathBuf::from(&args[9]);
|
|
fs::create_dir(&output)?;
|
|
let mut provenance = BTreeMap::new();
|
|
for file in [
|
|
"crates/searchi-speech/src/lib.rs",
|
|
"crates/searchi-speech/src/process.rs",
|
|
"crates/searchi-speech/src/inventory.rs",
|
|
"crates/searchi-service/src/speech.rs",
|
|
"crates/searchi-service/src/media.rs",
|
|
"crates/searchi-service/src/store.rs",
|
|
"tools/speech-runtime/worker.cpp",
|
|
] {
|
|
provenance.insert(file, file_hash(Path::new(file))?);
|
|
}
|
|
write_new(
|
|
&output.join("preflight.json"),
|
|
&json!({"fixture_sha256":hash(&fixture_raw),"prepared_sha256":hash(&prepared_raw),"binary_sha256":file_hash(&std::env::current_exe()?)?,"speech_worker_sha256":file_hash(&assets.speech)?,"speech_inventory_sha256":file_hash(&assets.speech.parent().context("Runtime directory")?.join("runtime-manifest.json"))?,"base_sha256":file_hash(&assets.base)?,"vad_sha256":file_hash(&assets.vad)?,"google_worker_sha256":file_hash(&assets.google)?,"source_hashes_at_start":provenance,"speech_policy":searchi_speech::POLICY,"source_clock_policy":"ActualService speech30s overlap2s; independently decodedPCM window hashes; no strict word labels","force_cpu":true,"threads":1}),
|
|
)?;
|
|
let core = prepared["clips"].as_array().context("Core clips missing")?;
|
|
ensure!(core.len() == 4, "All four controls required");
|
|
let queries = fixture["queries"].as_array().context("Queries missing")?;
|
|
ensure!(queries.len() == 10, "All ten queries required");
|
|
let first = match run_case("core", core, queries, &assets, &output) {
|
|
Ok(value) => value,
|
|
Err(error) => {
|
|
write_new(
|
|
&output.join("failure.json"),
|
|
&json!({"complete":false,"case":"core","error":format!("{error:#}"),"comparison_eligible":false}),
|
|
)?;
|
|
return Err(error);
|
|
}
|
|
};
|
|
let seams = prepared["seam_controls"]
|
|
.as_array()
|
|
.context("Seam controls missing")?;
|
|
ensure!(seams.len() == 2, "Both seam sources required");
|
|
let second = match run_case("seams", seams, &[], &assets, &output) {
|
|
Ok(value) => value,
|
|
Err(error) => {
|
|
write_new(
|
|
&output.join("failure.json"),
|
|
&json!({"complete":false,"case":"seams","error":format!("{error:#}"),"comparison_eligible":false}),
|
|
)?;
|
|
return Err(error);
|
|
}
|
|
};
|
|
write_new(
|
|
&output.join("report.json"),
|
|
&json!({"complete":true,"core":first,"seams":second,"strict_point_score_eligible":false,"production_policy_changed":false,"scope":"Actual shared native SpeechWorker/service/Google path; independent ENDE tinycorpus; recognized outputs immutable acceptedAPIvalues, no rawwire capture; full-sourceVAD comparator not executed; seam source/window bounds only"}),
|
|
)
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
#[test]
|
|
fn overlap_maps_complete_seam_tail() {
|
|
let value = windows(0, 40024).unwrap();
|
|
assert_eq!(value.len(), 2);
|
|
assert_eq!(value[1].start_ms, 28000);
|
|
assert_eq!(value[1].duration_ms, 12024);
|
|
}
|
|
#[test]
|
|
fn positioned_short_source_keeps_origin() {
|
|
let value = windows(2000, 4000).unwrap();
|
|
assert_eq!(value[0].start_ms, 2000);
|
|
assert_eq!(value[0].duration_ms, 2000);
|
|
}
|
|
#[test]
|
|
fn invalid_or_large_fixture_ranges_fail() {
|
|
assert!(windows(3, 2).is_err());
|
|
assert!(windows(0, 200000).is_err());
|
|
}
|
|
#[test]
|
|
fn existing_raw_output_is_preserved() -> Result<()> {
|
|
let dir = tempfile::tempdir()?;
|
|
let path = dir.path().join("raw.json");
|
|
fs::write(&path, b"old")?;
|
|
assert!(write_new(&path, &json!({})).is_err());
|
|
ensure!(fs::read(path)? == b"old", "Changed report");
|
|
Ok(())
|
|
}
|
|
}
|