diff --git a/tools/media-eval/README.md b/tools/media-eval/README.md index fcd7a0b..88ca214 100644 --- a/tools/media-eval/README.md +++ b/tools/media-eval/README.md @@ -60,3 +60,21 @@ positives are explicitly ineligible, filters remain explicitly unsupported, and all unfiltered negatives remain visible without abstention. Existing queries are diagnostics rather than new held-out evidence. Unseen music with missing and misleading tags needs independently frozen validation before deployment. + +`music_fields` is the subsequent diagnostic hypothesis: individual whitelisted +fields use the neutral document title `File metadata` and a shared 2048-byte +budget. It reuses per-query audio maxima verbatim from the exact preserved v1 +report (SHA-256 checked), avoiding duplicate audio inference. All queries and +negatives remain; scores for every field are retained, with no threshold/weights. + +```powershell +cargo test --manifest-path tools/media-eval/Cargo.toml --bin music_fields --locked +cargo run --manifest-path tools/media-eval/Cargo.toml --bin music_fields --locked -- prepare .searchi/qa/music-metadata-v1-results.json .searchi/qa/music-fields-v2-plan.json +$env:SEARCHI_FORCE_CPU = '1' +cargo run --manifest-path tools/media-eval/Cargo.toml --bin music_fields --locked -- run fixtures/media .searchi/qa/music-metadata-v1-results.json .searchi/qa/music-fields-v2-plan.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/qa/music-fields-v2-results.json +``` + +Prepare before inference, preserve the plan hash, and use fresh output paths. +Runtime/worker and source hashes must match v1. This diagnostic intentionally +depends on its exact baseline and remains a known-pool hypothesis comparison; +it is not a general production scorer or new independent validation. diff --git a/tools/media-eval/src/bin/music_fields.rs b/tools/media-eval/src/bin/music_fields.rs new file mode 100644 index 0000000..f5715ca --- /dev/null +++ b/tools/media-eval/src/bin/music_fields.rs @@ -0,0 +1,216 @@ +//! Known-pool hypothesis: separate real metadata fields, retaining verified v1 audio scores. +use anyhow::{ensure, Context, Result}; +use searchi_semantic::{Device, GemmaEmbedder}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{fs, io::Write, path::PathBuf, time::Instant}; + +const BASELINE: &str = "f985ba639a14b8c0288b8fa7812d580f179f31c8cad0c21d67531c8a4fddc5b6"; +const POLICY: &str = "metadata-fields-v2:artist-title-album-genre-composer:2048bytes-total:neutral-file-title:max-real-evidence:no-cutoff"; +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn read_baseline(bytes: &[u8]) -> Result { + ensure!( + digest(bytes) == BASELINE, + "Expected exact preserved v1 report" + ); + let value: Value = serde_json::from_slice(bytes)?; + ensure!( + value["worker"]["model"] == "google/embeddinggemma-2" && value["worker"]["device"] == "cpu", + "Unexpected baseline runtime" + ); + Ok(value) +} +fn fields(tags: &Value) -> Vec { + let mut remaining = 2048usize; + let mut result = Vec::new(); + for field in ["artist", "title", "album", "genre", "composer"] { + if let Some(value) = tags[field].as_str().filter(|s| !s.trim().is_empty()) { + let prefix = format!("{field}: "); + let mut end = value.len().min(remaining.saturating_sub(prefix.len())); + while !value.is_char_boundary(end) { + end -= 1; + } + if end == 0 { + break; + } + let text = format!("{prefix}{}", &value[..end]); + remaining -= text.len(); + result.push(json!({"field":field,"text":text})); + } + } + result +} +fn write_new(path: &PathBuf, value: &Value) -> Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?; + file.write_all(&serde_json::to_vec_pretty(value)?)?; + file.sync_all()?; + Ok(()) +} +fn cosine(a: &[f32], b: &[f32]) -> Result { + ensure!(a.len() == 768 && b.len() == 768, "Expected 768 dimensions"); + ensure!(a.iter().chain(b).all(|v| v.is_finite()), "Nonfinite vector"); + Ok(a.iter() + .zip(b) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum()) +} +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + ensure!( + args.len() == 3 || args.len() == 7, + "prepare BASELINE PLAN or run CORPUS BASELINE PLAN MODEL WORKER OUTPUT" + ); + let preparing = args[0] == "prepare"; + ensure!( + (preparing && args.len() == 3) || (args[0] == "run" && args.len() == 7), + "Invalid arguments" + ); + let baseline_path = if preparing { &args[1] } else { &args[2] }; + let baseline = read_baseline(&fs::read(baseline_path)?)?; + let assets:Vec<_>=baseline["plan"]["assets"].as_array().context("Assets missing")?.iter().map(|asset|json!({"id":asset["id"],"path":asset["path"],"sha256":asset["sha256"],"fields":fields(&asset["all_actual_tags"])})).collect(); + let plan = json!({"policy":POLICY,"baseline_sha256":BASELINE,"fixture_sha256":baseline["plan"]["fixture_sha256"],"worker_sha256":baseline["worker_sha256"],"assets":assets,"queries":baseline["plan"]["queries"],"scope":"Known two-file diagnostic only. Audio maxima reused verbatim from hash-verified v1. All queries retained; no cutoff or weights. Other-file positives ineligible. Metadata never receives a timestamp and misleading tags remain unverified."}); + let plan_path = PathBuf::from(if preparing { &args[2] } else { &args[3] }); + if preparing { + return write_new(&plan_path, &plan); + } + ensure!( + std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"), + "CPU required" + ); + let frozen = fs::read(&plan_path)?; + ensure!( + serde_json::from_slice::(&frozen)? == plan, + "Frozen plan changed" + ); + let root = PathBuf::from(&args[1]); + ensure!( + plan["fixture_sha256"] == digest(&fs::read(root.join("fixture.json"))?), + "Fixture changed" + ); + for asset in &assets { + ensure!( + asset["sha256"] + == digest(&fs::read( + root.join(asset["path"].as_str().context("Path missing")?) + )?), + "Asset changed" + ); + } + ensure!( + plan["worker_sha256"] == digest(&fs::read(&args[5])?), + "Worker differs from retained audio baseline" + ); + let output = PathBuf::from(&args[6]); + ensure!(!output.exists(), "Preserve existing output"); + let mut model = GemmaEmbedder::load_with_device(&args[4], &args[5], 1, Device::Cpu)?; + ensure!( + serde_json::to_value(&model.info)? == baseline["worker"], + "Runtime differs from baseline" + ); + let started = Instant::now(); + let mut evidence = Vec::new(); + for asset in &assets { + for field in asset["fields"].as_array().context("Fields missing")? { + let text = field["text"].as_str().context("Field text missing")?; + let mut vectors = model.embed_documents(&[text], "File metadata")?; + ensure!(vectors.len() == 1, "Expected one field vector"); + evidence.push(( + json!({"asset":asset["id"],"kind":"metadata","field":field["field"],"text":text}), + vectors.remove(0), + )); + } + } + let encoding_ms = started.elapsed().as_secs_f64() * 1000.0; + let mut rows = Vec::new(); + for previous in baseline["rows"].as_array().context("Rows missing")? { + if previous.get("unsupported").is_some() { + rows.push(previous.clone()); + continue; + } + let query = &previous["query"]; + let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?; + let mut ranked = previous["lanes"][0]["file_rankings"] + .as_array() + .context("Audio maxima missing")? + .clone(); + let mut field_scores = Vec::new(); + for (source, encoded) in &evidence { + let score = cosine(&vector, encoded)?; + field_scores.push(json!({"source":source,"score":score})); + let hit = ranked + .iter_mut() + .find(|hit| hit["source"]["asset"] == source["asset"]) + .context("Missing retained file")?; + if score > hit["score"].as_f64().context("Score missing")? { + *hit = json!({"source":source,"score":score}); + } + } + ranked.sort_by(|a, b| { + b["score"] + .as_f64() + .unwrap() + .total_cmp(&a["score"].as_f64().unwrap()) + }); + rows.push(json!({"query":query,"pool_quality_eligible":previous["pool_quality_eligible"],"audio_only":previous["lanes"][0]["file_rankings"],"whole_metadata_v1":previous["lanes"][1]["file_rankings"],"field_max_v2":ranked,"field_scores":field_scores})); + } + for asset in &assets { + ensure!( + asset["sha256"] == digest(&fs::read(root.join(asset["path"].as_str().unwrap()))?), + "Asset changed during inference" + ); + } + write_new( + &output, + &json!({"plan":plan,"plan_sha256":digest(&frozen),"worker":model.info,"encoding_ms":encoding_ms,"metadata_inference_calls":evidence.len(),"audio_inference_calls":0,"rows":rows,"production_policy_changed":false}), + ) +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn untrusted_boilerplate_and_blank_fields_are_excluded() { + assert!(fields(&json!({"encoder":"violin","title":" "})).is_empty()); + } + #[test] + fn field_order_and_unicode_budget_are_stable() { + let values = fields(&json!({"title":"รค".repeat(2000),"artist":"Actual"})); + assert_eq!(values[0]["field"], "artist"); + assert!( + values + .iter() + .map(|v| v["text"].as_str().unwrap().len()) + .sum::() + <= 2048 + ); + } + #[test] + fn misleading_metadata_remains_literal() { + assert_eq!( + fields(&json!({"genre":"Wrong genre"}))[0]["text"], + "genre: Wrong genre" + ); + } + #[test] + fn baseline_tampering_is_rejected() { + assert!(read_baseline(b"{}").is_err()); + } + #[test] + fn dimension_mismatch_and_nonfinite_vectors_are_rejected() { + assert!(cosine(&[1.0], &[1.0]).is_err()); + assert!(cosine(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err()); + } + #[test] + fn output_is_never_overwritten() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("report"); + fs::write(&path, b"old")?; + assert!(write_new(&path, &json!({})).is_err()); + assert_eq!(fs::read(path)?, b"old"); + Ok(()) + } +}