Freeze actual Google music metadata and precise speech timing diagnostics

tools/media-eval/Cargo.lock

@@ -869,11 +869,16 @@ name = "searchi-core"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "libc",
  "lopdf",
  "quick-xml",
  "rusqlite",
+ "searchi-media",
  "serde",
+ "serde_json",
+ "sha2 0.10.9",
  "walkdir",
+ "windows-sys",
  "zip",
 ]
 

tools/media-eval/README.md

@@ -33,3 +33,30 @@ The dense reference samples video every two seconds and PCM in thirty-second win
 - Model agreement with another implementation, synthetic document-seed scores, and this small-media benchmark each answer different questions. None is production quality certification.
 
 Detailed provenance is in [the fixture README](../../fixtures/media/README.md). The final assessment belongs in [MEDIA-QUALITY.md](../../docs/MEDIA-QUALITY.md).
+
+## Music metadata hypothesis diagnostic
+
+`music_metadata` compares raw audio with the per-file maximum over real audio
+and one Google-embedded metadata passage. It has no tuned weight or rejection
+threshold and does not change production. `prepare` freezes actual FFprobe tags,
+source hashes, default production audio windows, and all 65 existing queries
+before inference. Existing plans/reports cannot be overwritten.
+
+```powershell
+cargo test --manifest-path tools/media-eval/Cargo.toml --bin music_metadata --locked
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin music_metadata --locked -- prepare fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-metadata-v1-plan.json
+$env:SEARCHI_FORCE_CPU = '1'
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin music_metadata --locked -- run fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-metadata-v1-plan.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/qa/music-metadata-v1-results.json
+```
+
+The whitelist is artist, title, album, genre and composer in that fixed order,
+bounded to 2048 UTF-8 bytes. Missing/blank tags produce no metadata vector;
+encoder, comment, URLs and copyright boilerplate are excluded. Literal misleading
+tags remain misleading evidence: this tool does not verify musical instruments
+or generate a description. Metadata evidence has no invented timestamp.
+
+This is the known two-music-file pool, not mixed production retrieval. Other-file
+positives are explicitly ineligible, filters remain explicitly unsupported, and
+all unfiltered negatives remain visible without abstention. Existing queries are
+diagnostics rather than new held-out evidence. Unseen music with missing and
+misleading tags needs independently frozen validation before deployment.

tools/media-eval/src/bin/music_metadata.rs

@@ -0,0 +1,224 @@
+//! Frozen two-file hypothesis diagnostic; not production or abstention evidence.
+use anyhow::{ensure, Context, Result};
+use searchi_media::{audio_windows, decode_audio, probe, FfmpegTools, MediaLimits};
+use searchi_semantic::{Device, GemmaEmbedder};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{collections::BTreeMap, fs, io::Write, path::PathBuf, time::Instant};
+
+const POLICY: &str =
+    "music-metadata-v1:artist-title-album-genre-composer:2048bytes:max-real-evidence:no-cutoff";
+fn digest(bytes: &[u8]) -> String {
+    format!("{:x}", Sha256::digest(bytes))
+}
+fn metadata_text(tags: &BTreeMap<String, String>) -> String {
+    let mut text = String::new();
+    for key in ["artist", "title", "album", "genre", "composer"] {
+        if let Some(value) = tags.get(key).filter(|v| !v.trim().is_empty()) {
+            let prefix = format!("{key}: ");
+            let available = 2048usize.saturating_sub(text.len() + prefix.len() + 1);
+            let mut end = value.len().min(available);
+            while !value.is_char_boundary(end) {
+                end -= 1;
+            }
+            if end == 0 {
+                break;
+            }
+            text.push_str(&prefix);
+            text.push_str(&value[..end]);
+            text.push('\n');
+        }
+    }
+    text
+}
+fn write_new(path: &PathBuf, value: &Value) -> Result<()> {
+    let mut file = fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)?;
+    file.write_all(&serde_json::to_vec_pretty(value)?)?;
+    file.sync_all()?;
+    Ok(())
+}
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 5 || args.len() == 8,
+        "prepare CORPUS FFMPEG FFPROBE PLAN or run CORPUS FFMPEG FFPROBE PLAN MODEL WORKER OUTPUT"
+    );
+    let mode = args[0].to_string_lossy();
+    ensure!(
+        (mode == "prepare" && args.len() == 5) || (mode == "run" && args.len() == 8),
+        "Invalid mode/arguments"
+    );
+    let root = PathBuf::from(&args[1]);
+    let fixture_bytes = fs::read(root.join("fixture.json"))?;
+    let fixture: Value = serde_json::from_slice(&fixture_bytes)?;
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(
+        lock["fixture_sha256"] == digest(&fixture_bytes),
+        "Frozen fixture changed"
+    );
+    let tools = FfmpegTools::new(PathBuf::from(&args[2]), PathBuf::from(&args[3]))?;
+    let limits = MediaLimits::default();
+    let mut assets = Vec::new();
+    let mut probes = Vec::new();
+    for asset in fixture["assets"]
+        .as_array()
+        .context("Assets missing")?
+        .iter()
+        .filter(|a| a["kind"] == "music")
+    {
+        let path = root.join(asset["path"].as_str().context("Path missing")?);
+        ensure!(
+            asset["sha256"] == digest(&fs::read(&path)?),
+            "Music bytes changed"
+        );
+        let metadata = probe(&tools, &path, &limits)?;
+        assets.push(json!({"id":asset["id"],"path":asset["path"],"sha256":asset["sha256"],"metadata_text":metadata_text(&metadata.tags),"all_actual_tags":metadata.tags,"audio_windows":audio_windows(&metadata,&limits)?}));
+        probes.push((path, metadata));
+    }
+    ensure!(
+        assets.len() == 2,
+        "This diagnostic requires the known two-file pool"
+    );
+    let plan = json!({"policy":POLICY,"fixture_sha256":digest(&fixture_bytes),"assets":assets,"queries":fixture["queries"],"scope":"All frozen queries retained; other-file positives and unsupported filters are not quality eligible. Raw audio versus per-file maximum of raw audio and real metadata; no threshold, no weights. Misleading tags are not verified musical descriptions; empty tags add no evidence."});
+    let plan_path = PathBuf::from(&args[4]);
+    if mode == "prepare" {
+        return write_new(&plan_path, &plan);
+    }
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU required"
+    );
+    let frozen_bytes = fs::read(&plan_path)?;
+    ensure!(
+        serde_json::from_slice::<Value>(&frozen_bytes)? == plan,
+        "Prepared inputs or policy changed"
+    );
+    let output = PathBuf::from(&args[7]);
+    ensure!(!output.exists(), "Existing output must be preserved");
+    let mut model = GemmaEmbedder::load_with_device(&args[5], &args[6], 1, Device::Cpu)?;
+    ensure!(
+        model.info.device.to_lowercase().contains("cpu"),
+        "Worker is not CPU"
+    );
+    let started = Instant::now();
+    let mut evidence = Vec::new();
+    for (asset, (path, metadata)) in assets.iter().zip(&probes) {
+        for window in audio_windows(metadata, &limits)? {
+            let vector = model.embed_audio_pcm(&decode_audio(&tools, path, window, &limits)?)?;
+            evidence.push((json!({"asset":asset["id"],"kind":"audio","start_ms":window.start_ms,"duration_ms":window.duration_ms}),vector));
+        }
+        let text = asset["metadata_text"]
+            .as_str()
+            .context("Metadata missing")?;
+        if !text.is_empty() {
+            let vectors = model.embed_documents(&[text], "Music file metadata")?;
+            ensure!(vectors.len() == 1, "Expected one metadata vector");
+            evidence.push((
+                json!({"asset":asset["id"],"kind":"metadata","text":text}),
+                vectors[0].clone(),
+            ));
+        }
+        ensure!(
+            asset["sha256"] == digest(&fs::read(path)?),
+            "Source changed during inference"
+        );
+        println!("Encoded {}", asset["id"]);
+    }
+    let encoding_ms = started.elapsed().as_secs_f64() * 1000.0;
+    let mut rows = Vec::new();
+    for query in fixture["queries"].as_array().context("Queries missing")? {
+        if query.get("filters").is_some() {
+            rows.push(json!({"query":query,"unsupported":"Pool-only diagnostic does not enforce production filters"}));
+            continue;
+        }
+        let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?;
+        let scores: Vec<_> = evidence
+            .iter()
+            .map(|(source, encoded)| {
+                (
+                    source,
+                    vector
+                        .iter()
+                        .zip(encoded)
+                        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+                        .sum::<f64>(),
+                )
+            })
+            .collect();
+        let mut lanes = Vec::new();
+        for metadata in [false, true] {
+            let mut files = Vec::new();
+            for asset in &assets {
+                let best = scores
+                    .iter()
+                    .filter(|(source, _)| {
+                        source["asset"] == asset["id"] && (metadata || source["kind"] == "audio")
+                    })
+                    .max_by(|a, b| a.1.total_cmp(&b.1))
+                    .context("Missing audio evidence")?;
+                files.push(json!({"source":best.0,"score":best.1}));
+            }
+            files.sort_by(|a, b| {
+                b["score"]
+                    .as_f64()
+                    .unwrap()
+                    .total_cmp(&a["score"].as_f64().unwrap())
+            });
+            lanes.push(json!({"metadata_enabled":metadata,"file_rankings":files}));
+        }
+        let eligible = query["expected"].as_array().is_some_and(|a| {
+            a.iter()
+                .all(|id| assets.iter().any(|asset| asset["id"] == *id))
+        });
+        rows.push(json!({"query":query,"pool_quality_eligible":eligible,"lanes":lanes}));
+    }
+    write_new(
+        &output,
+        &json!({"plan":plan,"plan_sha256":digest(&frozen_bytes),"worker":model.info,"worker_sha256":digest(&fs::read(&args[6])?),"encoding_ms":encoding_ms,"evidence_count":evidence.len(),"rows":rows,"production_policy_changed":false}),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn empty_and_untrusted_boilerplate_add_no_evidence() {
+        assert!(metadata_text(&BTreeMap::from([
+            ("encoder".into(), "Violin".into()),
+            ("artist".into(), "  ".into())
+        ]))
+        .is_empty());
+    }
+    #[test]
+    fn unicode_is_bounded_without_inventing_descriptions() {
+        let text = metadata_text(&BTreeMap::from([
+            ("title".into(), "ä".repeat(3000)),
+            ("genre".into(), "Jazz".into()),
+        ]));
+        assert!(text.len() <= 2048);
+        assert!(text.starts_with("title: ä"));
+        assert!(!text.contains("Jazz"));
+    }
+    #[test]
+    fn stable_whitelist_order_and_literal_misleading_values() {
+        assert_eq!(
+            metadata_text(&BTreeMap::from([
+                ("artist".into(), "A".into()),
+                ("title".into(), "Not actually a violin".into())
+            ])),
+            "artist: A\ntitle: Not actually a violin\n"
+        );
+    }
+    #[test]
+    fn existing_report_is_preserved() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let path = dir.path().join("report.json");
+        fs::write(&path, b"original")?;
+        assert!(write_new(&path, &json!({})).is_err());
+        assert_eq!(fs::read(path)?, b"original");
+        Ok(())
+    }
+}