618 lines
29 KiB
Rust
618 lines
29 KiB
Rust
//! Genuine Google retrieval over recognized ASR text; labels never select segments.
|
|
use anyhow::{ensure, Context, Result};
|
|
use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
|
|
use serde_json::{json, Value};
|
|
use sha2::{Digest, Sha256};
|
|
use std::{
|
|
collections::{BTreeMap, BTreeSet},
|
|
fs,
|
|
io::{Read, Write},
|
|
path::{Path, PathBuf},
|
|
time::Instant,
|
|
};
|
|
|
|
const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
|
|
const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"];
|
|
mod alignment;
|
|
mod gated;
|
|
const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times";
|
|
|
|
fn digest(path: &Path) -> Result<String> {
|
|
let mut file = fs::File::open(path)?;
|
|
let mut hash = Sha256::new();
|
|
let mut buffer = [0; 65536];
|
|
loop {
|
|
let n = file.read(&mut buffer)?;
|
|
if n == 0 {
|
|
break;
|
|
}
|
|
hash.update(&buffer[..n]);
|
|
}
|
|
Ok(format!("{:x}", hash.finalize()))
|
|
}
|
|
fn number(value: &Value) -> Result<u64> {
|
|
value
|
|
.as_u64()
|
|
.context("Expected unsigned integer milliseconds")
|
|
}
|
|
fn checkpoint(path: &Path, report: &Value) -> Result<()> {
|
|
let parent = path
|
|
.parent()
|
|
.filter(|p| !p.as_os_str().is_empty())
|
|
.unwrap_or(Path::new("."));
|
|
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
|
|
temporary.write_all(&serde_json::to_vec_pretty(report)?)?;
|
|
temporary.as_file().sync_all()?;
|
|
temporary.persist(path)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Reconstruct absolute segment coordinates from actual whisper output, never labels.
|
|
fn parsed_segments(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
|
|
let rows = raw["transcription"]
|
|
.as_array()
|
|
.context("Missing actual transcription")?;
|
|
ensure!(rows.len() <= 1000, "Oversized transcription");
|
|
let mut segments = Vec::new();
|
|
let mut previous = 0;
|
|
for row in rows {
|
|
let start = number(&row["offsets"]["from"])?;
|
|
let end = number(&row["offsets"]["to"])?;
|
|
let text = row["text"].as_str().context("Invalid recognized text")?;
|
|
ensure!(
|
|
start >= previous && start <= end && end <= duration && text.len() <= 4096,
|
|
"Invalid actual ASR segment"
|
|
);
|
|
previous = start;
|
|
if !text.trim().is_empty() {
|
|
segments.push(json!({"start_ms":offset.checked_add(start).context("Timestamp overflow")?,"end_ms":offset.checked_add(end).context("Timestamp overflow")?,"text":text}));
|
|
}
|
|
}
|
|
Ok(segments)
|
|
}
|
|
|
|
fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
|
|
ensure!(
|
|
asr["version"] == 1 && asr["upstream_commit"] == "d1be6fde11ac6e0407606b4e42fe72d34add8037",
|
|
"Unknown ASR implementation"
|
|
);
|
|
ensure!(
|
|
matches!(
|
|
(asr["candidate"].as_str(), asr["model_sha256"].as_str()),
|
|
(
|
|
Some("base"),
|
|
Some("60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe")
|
|
) | (
|
|
Some("small"),
|
|
Some("1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b")
|
|
)
|
|
),
|
|
"Unknown ASR model"
|
|
);
|
|
ensure!(
|
|
asr["complete"] == true
|
|
&& asr["fixture_sha256"] == FIXTURE_SHA
|
|
&& matches!(asr["policy"].as_str(), Some(POLICY) | Some(gated::POLICY))
|
|
&& asr["backend"] == "forced CPU"
|
|
&& asr["threads"] == 1,
|
|
"Incomplete, stale, or unrecognized ASR report"
|
|
);
|
|
for field in [
|
|
"model_sha256",
|
|
"worker_sha256",
|
|
"harness_sha256",
|
|
"ffmpeg_sha256",
|
|
"ffprobe_sha256",
|
|
] {
|
|
ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance");
|
|
}
|
|
let token_alignment = alignment::enabled(asr)?;
|
|
let vad = gated::verified_report(asr)?;
|
|
let rows = asr["rows"].as_array().context("Missing ASR rows")?;
|
|
ensure!(rows.len() == 5, "All five competing assets required");
|
|
let mut seen = BTreeSet::new();
|
|
let mut segments = Vec::new();
|
|
let mut excluded = Vec::new();
|
|
for row in rows {
|
|
let asset_id = row["asset"].as_str().context("Missing asset ID")?;
|
|
ensure!(
|
|
ASSETS.contains(&asset_id) && seen.insert(asset_id),
|
|
"Unknown/duplicate asset"
|
|
);
|
|
let asset = fixture["assets"]
|
|
.as_array()
|
|
.context("Missing fixture assets")?
|
|
.iter()
|
|
.find(|asset| asset["id"] == asset_id)
|
|
.context("Unknown source")?;
|
|
ensure!(
|
|
row["source_sha256"] == asset["sha256"],
|
|
"Stale source provenance"
|
|
);
|
|
let start = number(&row["audio_start_ms"])?;
|
|
let end = number(&row["audio_end_ms"])?;
|
|
ensure!(
|
|
start < end && end - start <= 600000 && number(&row["duration_ms"])? == end - start,
|
|
"Unbounded audio range"
|
|
);
|
|
let windows = row["windows"].as_array().context("Missing windows")?;
|
|
ensure!(windows.len() <= 20, "Oversized window list");
|
|
let planned = vad
|
|
.as_ref()
|
|
.map(|vad| gated::planned_windows(vad, row, asset))
|
|
.transpose()?;
|
|
if let Some(planned) = &planned {
|
|
gated::verify_windows(windows, planned)?;
|
|
}
|
|
let mut next = start;
|
|
for (window_index, window) in windows.iter().enumerate() {
|
|
let offset = number(&window["offset_ms"])?;
|
|
let duration = number(&window["duration_ms"])?;
|
|
ensure!(
|
|
if let Some(planned) = &planned {
|
|
gated::matches_plan(window, &planned[window_index])
|
|
} else {
|
|
offset == next && duration == (end - offset).min(30000) && duration > 0
|
|
},
|
|
"Missing, overlapping or truncated source windows"
|
|
);
|
|
next = offset.checked_add(duration).context("Window overflow")?;
|
|
if let Some(error) = window.get("validation_error") {
|
|
ensure!(
|
|
error.as_str().is_some_and(|error| !error.is_empty()),
|
|
"Invalid exclusion reason"
|
|
);
|
|
if window["error_stage"] == "decode_coverage" {
|
|
let frames = number(&window["decoded_frames"])?;
|
|
ensure!(
|
|
window["asr_not_run"] == true
|
|
&& (frames > duration * 16
|
|
|| frames.saturating_add(16) < duration * 16),
|
|
"False decode exclusion"
|
|
);
|
|
} else {
|
|
ensure!(
|
|
parsed_segments(&window["actual_whisper_json"], offset, duration).is_err(),
|
|
"Valid recognized text must not be excluded"
|
|
);
|
|
}
|
|
excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error}));
|
|
continue;
|
|
}
|
|
let frames = number(&window["decoded_frames"])?;
|
|
ensure!(
|
|
frames <= duration * 16 && frames + 16 >= duration * 16,
|
|
"Stale/truncated decoded audio coverage"
|
|
);
|
|
let actual = parsed_segments(&window["actual_whisper_json"], offset, duration)?;
|
|
ensure!(
|
|
window["segments"] == json!(actual),
|
|
"Normalized ASR segments differ from actual output"
|
|
);
|
|
let actual = if token_alignment {
|
|
let dtw = asr["alignment_policy"] == alignment::DTW_POLICY;
|
|
if dtw {
|
|
let ordinary_failed =
|
|
alignment::reconstruct(&window["actual_whisper_json"], offset, duration)
|
|
.is_err();
|
|
ensure!(
|
|
window.get("ordinary_alignment_validation_error").is_some()
|
|
== ordinary_failed,
|
|
"Ordinary token failure evidence inconsistent"
|
|
);
|
|
if ordinary_failed {
|
|
ensure!(
|
|
window["ordinary_alignment_validation_error"]
|
|
.as_str()
|
|
.is_some_and(|error| !error.is_empty()),
|
|
"Invalid ordinary failure reason"
|
|
);
|
|
}
|
|
}
|
|
let rebuilt = alignment::reconstruct_mode(
|
|
&window["actual_whisper_json"],
|
|
offset,
|
|
duration,
|
|
dtw,
|
|
);
|
|
if let Some(error) = window.get("alignment_validation_error") {
|
|
ensure!(
|
|
error.as_str().is_some_and(|e| !e.is_empty()) && rebuilt.is_err(),
|
|
"Valid aligned text must not be excluded"
|
|
);
|
|
excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error}));
|
|
continue;
|
|
}
|
|
let rebuilt = rebuilt?;
|
|
ensure!(
|
|
window["aligned_segments"] == json!(rebuilt),
|
|
"Normalized token passages differ from actual output"
|
|
);
|
|
rebuilt
|
|
} else {
|
|
ensure!(
|
|
window.get("aligned_segments").is_none()
|
|
&& window.get("alignment_validation_error").is_none(),
|
|
"Alignment without provenance"
|
|
);
|
|
actual
|
|
};
|
|
let title = Path::new(asset["path"].as_str().context("Missing source filename")?)
|
|
.file_name()
|
|
.context("Missing filename")?
|
|
.to_string_lossy();
|
|
for (segment_index, segment) in actual.into_iter().enumerate() {
|
|
segments.push(json!({"asset":asset_id,"window_index":window_index,"segment_index":segment_index,"source_window_offset_ms":offset,"source_window_duration_ms":duration,"start_ms":segment["start_ms"],"end_ms":segment["end_ms"],"text":segment["text"],"title":title}));
|
|
ensure!(segments.len() <= 5000, "Too many recognized segments");
|
|
}
|
|
}
|
|
ensure!(
|
|
planned.is_some() || next == end,
|
|
"Incomplete source-window coverage"
|
|
);
|
|
}
|
|
ensure!(
|
|
asr["quality_failed"] == !excluded.is_empty(),
|
|
"ASR validation status inconsistent"
|
|
);
|
|
ensure!(
|
|
asr["validation_failures"]
|
|
.as_array()
|
|
.context("Missing validation failures")?
|
|
.len()
|
|
== excluded.len(),
|
|
"ASR validation failure count inconsistent"
|
|
);
|
|
if vad.is_some() {
|
|
ensure!(
|
|
asr["comparison_eligible"] == excluded.is_empty(),
|
|
"Gated comparison status inconsistent"
|
|
);
|
|
}
|
|
Ok((segments, excluded))
|
|
}
|
|
|
|
fn cosine(a: &[f32], b: &[f32]) -> Result<f64> {
|
|
ensure!(
|
|
a.len() == GEMMA_DIMENSIONS && b.len() == GEMMA_DIMENSIONS,
|
|
"Wrong vector dimensions"
|
|
);
|
|
let (mut dot, mut aa, mut bb) = (0.0f64, 0.0f64, 0.0f64);
|
|
for (&a, &b) in a.iter().zip(b) {
|
|
ensure!(a.is_finite() && b.is_finite(), "Nonfinite vector");
|
|
let (a, b) = (f64::from(a), f64::from(b));
|
|
dot += a * b;
|
|
aa += a * a;
|
|
bb += b * b;
|
|
}
|
|
ensure!(aa > 0.0 && bb > 0.0, "Zero vector");
|
|
Ok(dot / (aa * bb).sqrt())
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
|
ensure!(
|
|
args.len() == 5,
|
|
"Supply ASR_REPORT MEDIA_FIXTURE MODEL_DIR GOOGLE_WORKER NEW_OUTPUT_JSON"
|
|
);
|
|
ensure!(
|
|
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
|
"CPU only"
|
|
);
|
|
let asr_path = PathBuf::from(&args[0]);
|
|
let fixture_path = PathBuf::from(&args[1]);
|
|
let models = PathBuf::from(&args[2]);
|
|
let worker = PathBuf::from(&args[3]);
|
|
let output = PathBuf::from(&args[4]);
|
|
fs::OpenOptions::new()
|
|
.write(true)
|
|
.create_new(true)
|
|
.open(&output)
|
|
.context("Cannot reserve new output")?;
|
|
let mut report =
|
|
json!({"complete":false,"phase":"verification","rankings":{},"raw_queries":[]});
|
|
checkpoint(&output, &report)?;
|
|
ensure!(
|
|
fs::metadata(&asr_path)?.len() <= 16 * 1024 * 1024,
|
|
"Oversized ASR report"
|
|
);
|
|
let fixture_bytes = fs::read(&fixture_path)?;
|
|
ensure!(
|
|
format!("{:x}", Sha256::digest(&fixture_bytes)) == FIXTURE_SHA,
|
|
"Frozen media fixture changed"
|
|
);
|
|
let fixture: Value = serde_json::from_slice(&fixture_bytes)?;
|
|
let asr_bytes = fs::read(&asr_path)?;
|
|
let asr_hash = format!("{:x}", Sha256::digest(&asr_bytes));
|
|
let asr: Value = serde_json::from_slice(&asr_bytes)?;
|
|
let (segments, excluded) = eligible(&asr, &fixture)?;
|
|
ensure!(!segments.is_empty(), "No eligible recognized text");
|
|
for asset in fixture["assets"]
|
|
.as_array()
|
|
.context("Missing assets")?
|
|
.iter()
|
|
.filter(|asset| ASSETS.contains(&asset["id"].as_str().unwrap_or_default()))
|
|
{
|
|
let source = fixture_path
|
|
.parent()
|
|
.context("Missing fixture parent")?
|
|
.join(asset["path"].as_str().context("Missing asset path")?);
|
|
ensure!(
|
|
fs::metadata(&source)?.len() == number(&asset["bytes"])?
|
|
&& digest(&source)? == asset["sha256"],
|
|
"Frozen media source changed"
|
|
);
|
|
}
|
|
let manifest_path =
|
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json");
|
|
let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
|
|
for asset in manifest["files"]
|
|
.as_array()
|
|
.context("Missing model files")?
|
|
{
|
|
let path = models.join(asset["path"].as_str().context("Missing model path")?);
|
|
ensure!(
|
|
fs::metadata(&path)?.len() == number(&asset["bytes"])?
|
|
&& digest(&path)? == asset["sha256"],
|
|
"Google model bytes changed"
|
|
);
|
|
}
|
|
report["fixture_sha256"] = json!(FIXTURE_SHA);
|
|
report["asr_report_sha256"] = json!(asr_hash);
|
|
report["asr_provenance"] = asr.clone();
|
|
report["google_worker_sha256"] = json!(digest(&worker)?);
|
|
report["runner_sha256"] = json!(digest(&std::env::current_exe()?)?);
|
|
report["google_model_manifest_sha256"] = json!(digest(&manifest_path)?);
|
|
report["google_model"] = manifest;
|
|
report["segments"] = json!(segments);
|
|
report["excluded_windows"] = json!(excluded);
|
|
report["excluded_window_count"] = json!(excluded.len());
|
|
report["quality_score_eligible"] = json!(excluded.is_empty());
|
|
report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
|
|
if alignment::enabled(&asr)? {
|
|
report["scope"] = json!("Diagnostic token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate ordinary intervals or DTW point spans as declared in alignment_policy, no labels in ranking; not production integration.");
|
|
report["alignment_policy"] = asr["alignment_policy"].clone();
|
|
report["alignment_policy_sha256"] = asr["alignment_policy_sha256"].clone();
|
|
report["alignment_contract_sha256"] = asr["alignment_contract_sha256"].clone();
|
|
if asr.get("combination_policy").is_some() {
|
|
report["combination_policy"] = asr["combination_policy"].clone();
|
|
report["combination_policy_sha256"] = asr["combination_policy_sha256"].clone();
|
|
}
|
|
}
|
|
report["phase"] = json!("document_encoding");
|
|
checkpoint(&output, &report)?;
|
|
let start = Instant::now();
|
|
let mut model = GemmaEmbedder::load_with_device(&models, &worker, 1, Device::Cpu)?;
|
|
report["document_worker_info"] = json!(model.info);
|
|
let mut vectors = Vec::new();
|
|
let mut mappings = Vec::new();
|
|
for (segment_id, segment) in segments.iter().enumerate() {
|
|
let text = segment["text"].as_str().context("Missing text")?;
|
|
let title = segment["title"].as_str().context("Missing title")?;
|
|
for window in model.split_windows(text, title)? {
|
|
let mut encoded = model.embed_documents(&[&window.text], title)?;
|
|
ensure!(encoded.len() == 1, "Missing document embedding");
|
|
vectors.push(encoded.remove(0));
|
|
mappings.push(json!({"segment_id":segment_id,"text_start_byte":window.start_byte,"text_end_byte":window.end_byte}));
|
|
ensure!(vectors.len() <= 10000, "Too many token windows");
|
|
}
|
|
}
|
|
report["document_encoding_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
|
|
report["vector_source_mapping"] = json!(mappings);
|
|
drop(model);
|
|
let start = Instant::now();
|
|
let mut query_model = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
|
|
report["query_worker_info"] = json!(query_model.info);
|
|
report["query_startup_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
|
|
let queries: Vec<_> = fixture["queries"]
|
|
.as_array()
|
|
.context("Missing fixture queries")?
|
|
.iter()
|
|
.filter(|query| query["category"] == "video-speech")
|
|
.collect();
|
|
ensure!(
|
|
queries.len() == 8,
|
|
"Exactly eight frozen speech queries required"
|
|
);
|
|
report["phase"] = json!("query_ranking");
|
|
checkpoint(&output, &report)?;
|
|
for query in queries {
|
|
let start = Instant::now();
|
|
let vector =
|
|
query_model.embed_query(query["text"].as_str().context("Missing query text")?)?;
|
|
let mut raw = Vec::new();
|
|
let mut best: BTreeMap<usize, f64> = BTreeMap::new();
|
|
for (id, document) in vectors.iter().enumerate() {
|
|
let score = cosine(&vector, document)?;
|
|
let segment_id = number(&mappings[id]["segment_id"])? as usize;
|
|
best.entry(segment_id)
|
|
.and_modify(|old| *old = old.max(score))
|
|
.or_insert(score);
|
|
raw.push(json!({"vector_id":id,"segment_id":segment_id,"cosine":score}));
|
|
}
|
|
raw.sort_by(|a, b| {
|
|
b["cosine"]
|
|
.as_f64()
|
|
.unwrap()
|
|
.total_cmp(&a["cosine"].as_f64().unwrap())
|
|
.then_with(|| a["vector_id"].as_u64().cmp(&b["vector_id"].as_u64()))
|
|
});
|
|
let mut ranks: Vec<_> = best.into_iter().collect();
|
|
ranks.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
|
|
let all: Vec<_> = ranks.into_iter().map(|(id,score)| json!({"asset":segments[id]["asset"],"start_ms":segments[id]["start_ms"],"end_ms":segments[id]["end_ms"],"segment_id":id,"cosine":score})).collect();
|
|
report["rankings"][query["id"].as_str().context("Missing query ID")?] =
|
|
json!(all.iter().take(100).collect::<Vec<_>>());
|
|
report["raw_queries"].as_array_mut().context("Missing output queries")?.push(json!({"id":query["id"],"text":query["text"],"elapsed_ms":start.elapsed().as_secs_f64()*1000.0,"all_segment_ranks":all,"all_vector_ranks":raw}));
|
|
checkpoint(&output, &report)?;
|
|
}
|
|
ensure!(
|
|
report["asr_report_sha256"] == digest(&asr_path)?,
|
|
"ASR report changed during ranking"
|
|
);
|
|
ensure!(
|
|
digest(&fixture_path)? == FIXTURE_SHA
|
|
&& report["google_worker_sha256"] == digest(&worker)?
|
|
&& report["google_model_manifest_sha256"] == digest(&manifest_path)?,
|
|
"Fixture or runtime provenance changed during ranking"
|
|
);
|
|
report["complete"] = json!(true);
|
|
report["phase"] = json!("complete");
|
|
checkpoint(&output, &report)?;
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
#[test]
|
|
fn timestamps_are_absolute_milliseconds_and_invalid_tail_rejected() {
|
|
let raw =
|
|
json!({"transcription":[{"offsets":{"from":100,"to":2000},"text":" untouched text "}]});
|
|
assert_eq!(
|
|
parsed_segments(&raw, 30000, 3000).unwrap(),
|
|
vec![json!({"start_ms":30100,"end_ms":32000,"text":" untouched text "})]
|
|
);
|
|
assert!(parsed_segments(&raw, 30000, 1000).is_err());
|
|
assert!(parsed_segments(&raw, u64::MAX, 3000).is_err());
|
|
assert!(parsed_segments(
|
|
&json!({"transcription":[{"offsets":{"from":1.0,"to":2},"text":"x"}]}),
|
|
0,
|
|
30
|
|
)
|
|
.is_err());
|
|
}
|
|
#[test]
|
|
fn incomplete_and_stale_asr_are_rejected_before_inference() {
|
|
assert!(eligible(&json!({"complete":false}), &json!({})).is_err());
|
|
assert!(eligible(
|
|
&json!({"complete":true,"fixture_sha256":"stale"}),
|
|
&json!({})
|
|
)
|
|
.is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn frozen_gated_dtw_pool_requires_all_sources_windows_and_policy_bytes() {
|
|
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
|
let path = root.join(".searchi/speech/gated-base-results.json");
|
|
if !path.exists() {
|
|
return;
|
|
}
|
|
let fixture: Value =
|
|
serde_json::from_slice(&fs::read(root.join("fixtures/media/fixture.json")).unwrap())
|
|
.unwrap();
|
|
let mut report: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap();
|
|
let plan = include_str!("../../speech-vad/gated-dtw-policy.json");
|
|
report["combination_policy_utf8"] = json!(plan);
|
|
report["combination_policy"] = json!(alignment::COMBINATION_POLICY);
|
|
report["combination_policy_sha256"] = json!(alignment::COMBINATION_SHA);
|
|
report["alignment_policy"] = json!(alignment::DTW_POLICY);
|
|
report["alignment_policy_sha256"] = json!(alignment::DTW_POLICY_SHA);
|
|
report["alignment_contract_sha256"] = json!(alignment::DTW_CONTRACT_SHA);
|
|
// Explicitly synthetic token points test the coverage/protocol path,
|
|
// not actual alignment or quality. Original frozen VAD windows unchanged.
|
|
for row in report["rows"].as_array_mut().unwrap() {
|
|
for window in row["windows"].as_array_mut().unwrap() {
|
|
for segment in window["actual_whisper_json"]["transcription"]
|
|
.as_array_mut()
|
|
.unwrap()
|
|
{
|
|
segment["tokens"] = json!([{"id":100,"text":segment["text"],"offsets":segment["offsets"],"t_dtw":number(&segment["offsets"]["from"]).unwrap()/10}]);
|
|
}
|
|
window["aligned_segments"] = json!(alignment::reconstruct_mode(
|
|
&window["actual_whisper_json"],
|
|
number(&window["offset_ms"]).unwrap(),
|
|
number(&window["duration_ms"]).unwrap(),
|
|
true
|
|
)
|
|
.unwrap());
|
|
}
|
|
}
|
|
assert!(eligible(&report, &fixture).unwrap().1.is_empty());
|
|
let mut changed = report.clone();
|
|
changed["combination_policy_utf8"] = json!(format!("{plan}\n"));
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report.clone();
|
|
changed["rows"].as_array_mut().unwrap().pop();
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report.clone();
|
|
changed["rows"][0]["windows"].as_array_mut().unwrap().pop();
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report.clone();
|
|
let no_speech = changed["rows"]
|
|
.as_array_mut()
|
|
.unwrap()
|
|
.iter_mut()
|
|
.find(|row| row["status"] == "no_speech")
|
|
.unwrap();
|
|
no_speech["asr_not_run"] = json!(false);
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report;
|
|
changed["rows"][0]["windows"][0]["actual_whisper_json"]["transcription"][0]["tokens"][0]
|
|
["t_dtw"] = json!(-1);
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn aligned_pool_preserves_all_competitors_and_marks_real_failures_ineligible() {
|
|
let hash = "a".repeat(64);
|
|
let assets: Vec<_> = ASSETS
|
|
.iter()
|
|
.map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.wav")}))
|
|
.collect();
|
|
let raw = json!({"transcription":[{"text":" First. Second!","offsets":{"from":0,"to":1000},"tokens":[
|
|
{"id":100,"text":" First.","offsets":{"from":100,"to":400}},
|
|
{"id":101,"text":" Second!","offsets":{"from":600,"to":900}}]}]});
|
|
let rows:Vec<_> = ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap(),"aligned_segments":alignment::reconstruct(&raw,1000,1000).unwrap()}]})).collect();
|
|
let mut report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":"12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8","harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows,"alignment_policy":alignment::POLICY,"alignment_policy_sha256":alignment::POLICY_SHA,"alignment_contract_sha256":alignment::CONTRACT_SHA});
|
|
let fixture = json!({"assets":assets});
|
|
assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 10);
|
|
let mut forged = report.clone();
|
|
forged["rows"][0]["windows"][0]["aligned_segments"][0]["start_ms"] = json!(1500);
|
|
assert!(eligible(&forged, &fixture).is_err());
|
|
let window = &mut report["rows"][0]["windows"][0];
|
|
window["actual_whisper_json"]["transcription"][0]["tokens"][0]
|
|
.as_object_mut()
|
|
.unwrap()
|
|
.remove("offsets");
|
|
window["alignment_validation_error"] = json!("Missing token time");
|
|
window.as_object_mut().unwrap().remove("aligned_segments");
|
|
report["quality_failed"] = json!(true);
|
|
report["validation_failures"] = json!([{"error":"Missing token time"}]);
|
|
let (pool, excluded) = eligible(&report, &fixture).unwrap();
|
|
assert_eq!(pool.len(), 8);
|
|
assert_eq!(excluded.len(), 1);
|
|
// Main derives quality_score_eligible solely from excluded.is_empty().
|
|
assert!(!excluded.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn source_identity_coverage_and_recognized_text_cannot_be_forged() {
|
|
let hash = "a".repeat(64);
|
|
let assets: Vec<_> = ASSETS
|
|
.iter()
|
|
.map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.mp3")}))
|
|
.collect();
|
|
let raw = json!({"transcription":[{"offsets":{"from":10,"to":900},"text":"actual unit-test text"}]});
|
|
let rows:Vec<_>=ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap()}]})).collect();
|
|
let report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":hash,"harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows});
|
|
let fixture = json!({"assets":assets});
|
|
assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 5);
|
|
let mut changed = report.clone();
|
|
changed["rows"][0]["asset"] = json!("unknown");
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report.clone();
|
|
changed["rows"][0]["source_sha256"] = json!("stale");
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report.clone();
|
|
changed["rows"][0]["windows"][0]["segments"][0]["start_ms"] = json!(10);
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report.clone();
|
|
changed["rows"][0]["windows"][0]["decoded_frames"] = json!(8000);
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
let mut changed = report;
|
|
changed["rows"][0]["windows"][0]["validation_error"] = json!("hide a valid competitor");
|
|
assert!(eligible(&changed, &fixture).is_err());
|
|
}
|
|
}
|