Validate audio stream timelines and add genuine Google transcript diagnostics

tools/speech-rank/src/main.rs

@@ -0,0 +1,440 @@
+//! Genuine Google retrieval over recognized ASR text; labels never select segments.
+use anyhow::{ensure, Context, Result};
+use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    fs,
+    io::{Read, Write},
+    path::{Path, PathBuf},
+    time::Instant,
+};
+
+const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
+const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"];
+const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times";
+
+fn digest(path: &Path) -> Result<String> {
+    let mut file = fs::File::open(path)?;
+    let mut hash = Sha256::new();
+    let mut buffer = [0; 65536];
+    loop {
+        let n = file.read(&mut buffer)?;
+        if n == 0 {
+            break;
+        }
+        hash.update(&buffer[..n]);
+    }
+    Ok(format!("{:x}", hash.finalize()))
+}
+fn number(value: &Value) -> Result<u64> {
+    value
+        .as_u64()
+        .context("Expected unsigned integer milliseconds")
+}
+fn checkpoint(path: &Path, report: &Value) -> Result<()> {
+    let parent = path
+        .parent()
+        .filter(|p| !p.as_os_str().is_empty())
+        .unwrap_or(Path::new("."));
+    let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
+    temporary.write_all(&serde_json::to_vec_pretty(report)?)?;
+    temporary.as_file().sync_all()?;
+    temporary.persist(path)?;
+    Ok(())
+}
+
+/// Reconstruct absolute segment coordinates from actual whisper output, never labels.
+fn parsed_segments(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
+    let rows = raw["transcription"]
+        .as_array()
+        .context("Missing actual transcription")?;
+    ensure!(rows.len() <= 1000, "Oversized transcription");
+    let mut segments = Vec::new();
+    let mut previous = 0;
+    for row in rows {
+        let start = number(&row["offsets"]["from"])?;
+        let end = number(&row["offsets"]["to"])?;
+        let text = row["text"].as_str().context("Invalid recognized text")?;
+        ensure!(
+            start >= previous && start <= end && end <= duration && text.len() <= 4096,
+            "Invalid actual ASR segment"
+        );
+        previous = start;
+        if !text.trim().is_empty() {
+            segments.push(json!({"start_ms":offset.checked_add(start).context("Timestamp overflow")?,"end_ms":offset.checked_add(end).context("Timestamp overflow")?,"text":text}));
+        }
+    }
+    Ok(segments)
+}
+
+fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
+    ensure!(
+        asr["version"] == 1 && asr["upstream_commit"] == "d1be6fde11ac6e0407606b4e42fe72d34add8037",
+        "Unknown ASR implementation"
+    );
+    ensure!(
+        matches!(
+            (asr["candidate"].as_str(), asr["model_sha256"].as_str()),
+            (
+                Some("base"),
+                Some("60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe")
+            ) | (
+                Some("small"),
+                Some("1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b")
+            )
+        ),
+        "Unknown ASR model"
+    );
+    ensure!(
+        asr["complete"] == true
+            && asr["fixture_sha256"] == FIXTURE_SHA
+            && asr["policy"] == POLICY
+            && asr["backend"] == "forced CPU"
+            && asr["threads"] == 1,
+        "Incomplete, stale, or unrecognized ASR report"
+    );
+    for field in [
+        "model_sha256",
+        "worker_sha256",
+        "harness_sha256",
+        "ffmpeg_sha256",
+        "ffprobe_sha256",
+    ] {
+        ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance");
+    }
+    let rows = asr["rows"].as_array().context("Missing ASR rows")?;
+    ensure!(rows.len() == 5, "All five competing assets required");
+    let mut seen = BTreeSet::new();
+    let mut segments = Vec::new();
+    let mut excluded = Vec::new();
+    for row in rows {
+        let asset_id = row["asset"].as_str().context("Missing asset ID")?;
+        ensure!(
+            ASSETS.contains(&asset_id) && seen.insert(asset_id),
+            "Unknown/duplicate asset"
+        );
+        let asset = fixture["assets"]
+            .as_array()
+            .context("Missing fixture assets")?
+            .iter()
+            .find(|asset| asset["id"] == asset_id)
+            .context("Unknown source")?;
+        ensure!(
+            row["source_sha256"] == asset["sha256"],
+            "Stale source provenance"
+        );
+        let start = number(&row["audio_start_ms"])?;
+        let end = number(&row["audio_end_ms"])?;
+        ensure!(
+            start < end && end - start <= 600000 && number(&row["duration_ms"])? == end - start,
+            "Unbounded audio range"
+        );
+        let windows = row["windows"].as_array().context("Missing windows")?;
+        ensure!(windows.len() <= 20, "Oversized window list");
+        let mut next = start;
+        for (window_index, window) in windows.iter().enumerate() {
+            let offset = number(&window["offset_ms"])?;
+            let duration = number(&window["duration_ms"])?;
+            ensure!(
+                offset == next && duration == (end - offset).min(30000) && duration > 0,
+                "Missing, overlapping or truncated source windows"
+            );
+            next = offset.checked_add(duration).context("Window overflow")?;
+            if let Some(error) = window.get("validation_error") {
+                ensure!(
+                    error.as_str().is_some_and(|error| !error.is_empty()),
+                    "Invalid exclusion reason"
+                );
+                if window["error_stage"] == "decode_coverage" {
+                    let frames = number(&window["decoded_frames"])?;
+                    ensure!(
+                        window["asr_not_run"] == true
+                            && (frames > duration * 16
+                                || frames.saturating_add(16) < duration * 16),
+                        "False decode exclusion"
+                    );
+                } else {
+                    ensure!(
+                        parsed_segments(&window["actual_whisper_json"], offset, duration).is_err(),
+                        "Valid recognized text must not be excluded"
+                    );
+                }
+                excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error}));
+                continue;
+            }
+            let frames = number(&window["decoded_frames"])?;
+            ensure!(
+                frames <= duration * 16 && frames + 16 >= duration * 16,
+                "Stale/truncated decoded audio coverage"
+            );
+            let actual = parsed_segments(&window["actual_whisper_json"], offset, duration)?;
+            ensure!(
+                window["segments"] == json!(actual),
+                "Normalized ASR segments differ from actual output"
+            );
+            let title = Path::new(asset["path"].as_str().context("Missing source filename")?)
+                .file_name()
+                .context("Missing filename")?
+                .to_string_lossy();
+            for (segment_index, segment) in actual.into_iter().enumerate() {
+                segments.push(json!({"asset":asset_id,"window_index":window_index,"segment_index":segment_index,"source_window_offset_ms":offset,"source_window_duration_ms":duration,"start_ms":segment["start_ms"],"end_ms":segment["end_ms"],"text":segment["text"],"title":title}));
+                ensure!(segments.len() <= 5000, "Too many recognized segments");
+            }
+        }
+        ensure!(next == end, "Incomplete source-window coverage");
+    }
+    ensure!(
+        asr["quality_failed"] == !excluded.is_empty(),
+        "ASR validation status inconsistent"
+    );
+    ensure!(
+        asr["validation_failures"]
+            .as_array()
+            .context("Missing validation failures")?
+            .len()
+            == excluded.len(),
+        "ASR validation failure count inconsistent"
+    );
+    Ok((segments, excluded))
+}
+
+fn cosine(a: &[f32], b: &[f32]) -> Result<f64> {
+    ensure!(
+        a.len() == GEMMA_DIMENSIONS && b.len() == GEMMA_DIMENSIONS,
+        "Wrong vector dimensions"
+    );
+    let (mut dot, mut aa, mut bb) = (0.0f64, 0.0f64, 0.0f64);
+    for (&a, &b) in a.iter().zip(b) {
+        ensure!(a.is_finite() && b.is_finite(), "Nonfinite vector");
+        let (a, b) = (f64::from(a), f64::from(b));
+        dot += a * b;
+        aa += a * a;
+        bb += b * b;
+    }
+    ensure!(aa > 0.0 && bb > 0.0, "Zero vector");
+    Ok(dot / (aa * bb).sqrt())
+}
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 5,
+        "Supply ASR_REPORT MEDIA_FIXTURE MODEL_DIR GOOGLE_WORKER NEW_OUTPUT_JSON"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU only"
+    );
+    let asr_path = PathBuf::from(&args[0]);
+    let fixture_path = PathBuf::from(&args[1]);
+    let models = PathBuf::from(&args[2]);
+    let worker = PathBuf::from(&args[3]);
+    let output = PathBuf::from(&args[4]);
+    fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(&output)
+        .context("Cannot reserve new output")?;
+    let mut report =
+        json!({"complete":false,"phase":"verification","rankings":{},"raw_queries":[]});
+    checkpoint(&output, &report)?;
+    ensure!(
+        fs::metadata(&asr_path)?.len() <= 16 * 1024 * 1024,
+        "Oversized ASR report"
+    );
+    let fixture_bytes = fs::read(&fixture_path)?;
+    ensure!(
+        format!("{:x}", Sha256::digest(&fixture_bytes)) == FIXTURE_SHA,
+        "Frozen media fixture changed"
+    );
+    let fixture: Value = serde_json::from_slice(&fixture_bytes)?;
+    let asr_bytes = fs::read(&asr_path)?;
+    let asr_hash = format!("{:x}", Sha256::digest(&asr_bytes));
+    let asr: Value = serde_json::from_slice(&asr_bytes)?;
+    let (segments, excluded) = eligible(&asr, &fixture)?;
+    ensure!(!segments.is_empty(), "No eligible recognized text");
+    for asset in fixture["assets"]
+        .as_array()
+        .context("Missing assets")?
+        .iter()
+        .filter(|asset| ASSETS.contains(&asset["id"].as_str().unwrap_or_default()))
+    {
+        let source = fixture_path
+            .parent()
+            .context("Missing fixture parent")?
+            .join(asset["path"].as_str().context("Missing asset path")?);
+        ensure!(
+            fs::metadata(&source)?.len() == number(&asset["bytes"])?
+                && digest(&source)? == asset["sha256"],
+            "Frozen media source changed"
+        );
+    }
+    let manifest_path =
+        PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json");
+    let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
+    for asset in manifest["files"]
+        .as_array()
+        .context("Missing model files")?
+    {
+        let path = models.join(asset["path"].as_str().context("Missing model path")?);
+        ensure!(
+            fs::metadata(&path)?.len() == number(&asset["bytes"])?
+                && digest(&path)? == asset["sha256"],
+            "Google model bytes changed"
+        );
+    }
+    report["fixture_sha256"] = json!(FIXTURE_SHA);
+    report["asr_report_sha256"] = json!(asr_hash);
+    report["asr_provenance"] = asr.clone();
+    report["google_worker_sha256"] = json!(digest(&worker)?);
+    report["runner_sha256"] = json!(digest(&std::env::current_exe()?)?);
+    report["google_model_manifest_sha256"] = json!(digest(&manifest_path)?);
+    report["google_model"] = manifest;
+    report["segments"] = json!(segments);
+    report["excluded_windows"] = json!(excluded);
+    report["excluded_window_count"] = json!(excluded.len());
+    report["quality_score_eligible"] = json!(excluded.is_empty());
+    report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
+    report["phase"] = json!("document_encoding");
+    checkpoint(&output, &report)?;
+    let start = Instant::now();
+    let mut model = GemmaEmbedder::load_with_device(&models, &worker, 1, Device::Cpu)?;
+    report["document_worker_info"] = json!(model.info);
+    let mut vectors = Vec::new();
+    let mut mappings = Vec::new();
+    for (segment_id, segment) in segments.iter().enumerate() {
+        let text = segment["text"].as_str().context("Missing text")?;
+        let title = segment["title"].as_str().context("Missing title")?;
+        for window in model.split_windows(text, title)? {
+            let mut encoded = model.embed_documents(&[&window.text], title)?;
+            ensure!(encoded.len() == 1, "Missing document embedding");
+            vectors.push(encoded.remove(0));
+            mappings.push(json!({"segment_id":segment_id,"text_start_byte":window.start_byte,"text_end_byte":window.end_byte}));
+            ensure!(vectors.len() <= 10000, "Too many token windows");
+        }
+    }
+    report["document_encoding_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
+    report["vector_source_mapping"] = json!(mappings);
+    drop(model);
+    let start = Instant::now();
+    let mut query_model = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
+    report["query_worker_info"] = json!(query_model.info);
+    report["query_startup_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
+    let queries: Vec<_> = fixture["queries"]
+        .as_array()
+        .context("Missing fixture queries")?
+        .iter()
+        .filter(|query| query["category"] == "video-speech")
+        .collect();
+    ensure!(
+        queries.len() == 8,
+        "Exactly eight frozen speech queries required"
+    );
+    report["phase"] = json!("query_ranking");
+    checkpoint(&output, &report)?;
+    for query in queries {
+        let start = Instant::now();
+        let vector =
+            query_model.embed_query(query["text"].as_str().context("Missing query text")?)?;
+        let mut raw = Vec::new();
+        let mut best: BTreeMap<usize, f64> = BTreeMap::new();
+        for (id, document) in vectors.iter().enumerate() {
+            let score = cosine(&vector, document)?;
+            let segment_id = number(&mappings[id]["segment_id"])? as usize;
+            best.entry(segment_id)
+                .and_modify(|old| *old = old.max(score))
+                .or_insert(score);
+            raw.push(json!({"vector_id":id,"segment_id":segment_id,"cosine":score}));
+        }
+        raw.sort_by(|a, b| {
+            b["cosine"]
+                .as_f64()
+                .unwrap()
+                .total_cmp(&a["cosine"].as_f64().unwrap())
+                .then_with(|| a["vector_id"].as_u64().cmp(&b["vector_id"].as_u64()))
+        });
+        let mut ranks: Vec<_> = best.into_iter().collect();
+        ranks.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
+        let all: Vec<_> = ranks.into_iter().map(|(id,score)| json!({"asset":segments[id]["asset"],"start_ms":segments[id]["start_ms"],"end_ms":segments[id]["end_ms"],"segment_id":id,"cosine":score})).collect();
+        report["rankings"][query["id"].as_str().context("Missing query ID")?] =
+            json!(all.iter().take(100).collect::<Vec<_>>());
+        report["raw_queries"].as_array_mut().context("Missing output queries")?.push(json!({"id":query["id"],"text":query["text"],"elapsed_ms":start.elapsed().as_secs_f64()*1000.0,"all_segment_ranks":all,"all_vector_ranks":raw}));
+        checkpoint(&output, &report)?;
+    }
+    ensure!(
+        report["asr_report_sha256"] == digest(&asr_path)?,
+        "ASR report changed during ranking"
+    );
+    ensure!(
+        digest(&fixture_path)? == FIXTURE_SHA
+            && report["google_worker_sha256"] == digest(&worker)?
+            && report["google_model_manifest_sha256"] == digest(&manifest_path)?,
+        "Fixture or runtime provenance changed during ranking"
+    );
+    report["complete"] = json!(true);
+    report["phase"] = json!("complete");
+    checkpoint(&output, &report)?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn timestamps_are_absolute_milliseconds_and_invalid_tail_rejected() {
+        let raw =
+            json!({"transcription":[{"offsets":{"from":100,"to":2000},"text":" untouched text "}]});
+        assert_eq!(
+            parsed_segments(&raw, 30000, 3000).unwrap(),
+            vec![json!({"start_ms":30100,"end_ms":32000,"text":" untouched text "})]
+        );
+        assert!(parsed_segments(&raw, 30000, 1000).is_err());
+        assert!(parsed_segments(&raw, u64::MAX, 3000).is_err());
+        assert!(parsed_segments(
+            &json!({"transcription":[{"offsets":{"from":1.0,"to":2},"text":"x"}]}),
+            0,
+            30
+        )
+        .is_err());
+    }
+    #[test]
+    fn incomplete_and_stale_asr_are_rejected_before_inference() {
+        assert!(eligible(&json!({"complete":false}), &json!({})).is_err());
+        assert!(eligible(
+            &json!({"complete":true,"fixture_sha256":"stale"}),
+            &json!({})
+        )
+        .is_err());
+    }
+
+    #[test]
+    fn source_identity_coverage_and_recognized_text_cannot_be_forged() {
+        let hash = "a".repeat(64);
+        let assets: Vec<_> = ASSETS
+            .iter()
+            .map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.mp3")}))
+            .collect();
+        let raw = json!({"transcription":[{"offsets":{"from":10,"to":900},"text":"actual unit-test text"}]});
+        let rows:Vec<_>=ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap()}]})).collect();
+        let report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":hash,"harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows});
+        let fixture = json!({"assets":assets});
+        assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 5);
+        let mut changed = report.clone();
+        changed["rows"][0]["asset"] = json!("unknown");
+        assert!(eligible(&changed, &fixture).is_err());
+        let mut changed = report.clone();
+        changed["rows"][0]["source_sha256"] = json!("stale");
+        assert!(eligible(&changed, &fixture).is_err());
+        let mut changed = report.clone();
+        changed["rows"][0]["windows"][0]["segments"][0]["start_ms"] = json!(10);
+        assert!(eligible(&changed, &fixture).is_err());
+        let mut changed = report.clone();
+        changed["rows"][0]["windows"][0]["decoded_frames"] = json!(8000);
+        assert!(eligible(&changed, &fixture).is_err());
+        let mut changed = report;
+        changed["rows"][0]["windows"][0]["validation_error"] = json!("hide a valid competitor");
+        assert!(eligible(&changed, &fixture).is_err());
+    }
+}