Freeze unseen music stress corpus and independently verified gated DTW experiment
This commit is contained in:
karl committed
1 parent
5ab24518b4
commit
d27be302c8
14 files changed
+1305
-6
No files matched your search
@@ -0,0 +1,218 @@
|
||||
//! Mixed-pool actual metadata stress diagnostic; frozen labels never enter encodes.
|
||||
use anyhow::{ensure, Context, Result};
|
||||
use searchi_media::{audio_windows, decode_audio, probe, FfmpegTools, MediaLimits};
|
||||
use searchi_semantic::{Device, GemmaEmbedder};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{collections::BTreeMap, fs, io::Write, path::PathBuf, time::Instant};
|
||||
fn sha(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
fn fields(tags: &Value) -> Vec<Value> {
|
||||
let mut left = 2048usize;
|
||||
let mut values = Vec::new();
|
||||
for key in ["artist", "title", "album", "genre", "composer"] {
|
||||
if let Some(value) = tags[key].as_str().filter(|s| !s.trim().is_empty()) {
|
||||
let prefix = format!("{key}: ");
|
||||
let mut end = value.len().min(left.saturating_sub(prefix.len()));
|
||||
while !value.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if end == 0 {
|
||||
break;
|
||||
}
|
||||
let text = format!("{prefix}{}", &value[..end]);
|
||||
left -= text.len();
|
||||
values.push(json!({"field":key,"text":text}));
|
||||
}
|
||||
}
|
||||
values
|
||||
}
|
||||
fn dot(a: &[f32], b: &[f32]) -> Result<f64> {
|
||||
ensure!(
|
||||
a.len() == 768 && b.len() == 768 && a.iter().chain(b).all(|x| x.is_finite()),
|
||||
"Invalid vector"
|
||||
);
|
||||
Ok(a.iter()
|
||||
.zip(b)
|
||||
.map(|(a, b)| f64::from(*a) * f64::from(*b))
|
||||
.sum())
|
||||
}
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
ensure!(
|
||||
args.len() == 7,
|
||||
"Supply FIXTURE PREPARED MODEL WORKER FFMPEG FFPROBE OUTPUT"
|
||||
);
|
||||
ensure!(
|
||||
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
||||
"CPU only"
|
||||
);
|
||||
let fixture_path = PathBuf::from(&args[0]);
|
||||
let fixture_raw = fs::read(&fixture_path)?;
|
||||
let fixture: Value = serde_json::from_slice(&fixture_raw)?;
|
||||
let lock: Value =
|
||||
serde_json::from_slice(&fs::read(fixture_path.with_file_name("labels.lock.json"))?)?;
|
||||
ensure!(
|
||||
lock["fixture_sha256"] == sha(&fixture_raw),
|
||||
"Frozen labels changed"
|
||||
);
|
||||
let plan_raw = fs::read(&args[1])?;
|
||||
let plan: Value = serde_json::from_slice(&plan_raw)?;
|
||||
let input_lock: Value =
|
||||
serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
|
||||
ensure!(
|
||||
input_lock["prepared_plan_sha256"] == sha(&plan_raw)
|
||||
&& input_lock["fixture_sha256"] == sha(&fixture_raw),
|
||||
"Prepared plan differs from pre-inference input lock"
|
||||
);
|
||||
ensure!(
|
||||
plan["fixture_sha256"] == sha(&fixture_raw) && plan["fixture"] == fixture,
|
||||
"Prepared inputs mismatch"
|
||||
);
|
||||
ensure!(
|
||||
plan["ffmpeg_sha256"] == sha(&fs::read(&args[4])?)
|
||||
&& plan["ffprobe_sha256"] == sha(&fs::read(&args[5])?),
|
||||
"Decoder changed"
|
||||
);
|
||||
let output = PathBuf::from(&args[6]);
|
||||
ensure!(!output.exists(), "Preserve existing report");
|
||||
let candidates = plan["candidates"]
|
||||
.as_array()
|
||||
.context("Candidates missing")?;
|
||||
let tools = FfmpegTools::new(PathBuf::from(&args[4]), PathBuf::from(&args[5]))?;
|
||||
let limits = MediaLimits::default();
|
||||
let mut model = GemmaEmbedder::load_with_device(&args[2], &args[3], 1, Device::Cpu)?;
|
||||
ensure!(model.info.device == "cpu", "CPU required");
|
||||
let started = Instant::now();
|
||||
let mut media: BTreeMap<String, (Value, Vec<f32>)> = BTreeMap::new();
|
||||
let mut metadata = Vec::new();
|
||||
let mut decoded = BTreeMap::new();
|
||||
for candidate in candidates {
|
||||
let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?);
|
||||
ensure!(
|
||||
candidate["sha256"] == sha(&fs::read(&path)?),
|
||||
"Candidate bytes changed"
|
||||
);
|
||||
let id = candidate["id"].as_str().context("ID missing")?.to_string();
|
||||
if candidate["kind"] == "audio" {
|
||||
let actual = probe(&tools, &path, &limits)?;
|
||||
ensure!(
|
||||
serde_json::to_value(&actual.tags)? == candidate["actual_tags"],
|
||||
"Actual tags differ"
|
||||
);
|
||||
let window = audio_windows(&actual, &limits)?
|
||||
.into_iter()
|
||||
.next()
|
||||
.context("Audio absent")?;
|
||||
let pcm = decode_audio(&tools, &path, window, &limits)?;
|
||||
let bytes: Vec<_> = pcm.iter().flat_map(|s| s.to_le_bytes()).collect();
|
||||
let hash = sha(&bytes);
|
||||
if let Some(expected) = candidate["pcm_sha256"].as_str() {
|
||||
ensure!(hash == expected, "Prepared PCM changed");
|
||||
}
|
||||
if let Some(old) = decoded.insert(id.clone(), hash.clone()) {
|
||||
ensure!(old == hash, "Variants changed decoded audio");
|
||||
}
|
||||
if !media.contains_key(&id) {
|
||||
media.insert(id.clone(),(json!({"asset":id,"kind":"audio","path":path,"start_ms":0,"duration_ms":window.duration_ms}),model.embed_audio_pcm(&pcm)?));
|
||||
}
|
||||
} else if !media.contains_key(&id) {
|
||||
media.insert(
|
||||
id.clone(),
|
||||
(
|
||||
json!({"asset":id,"kind":"image","path":path}),
|
||||
model.embed_images_with_budget(&[&path], 280)?.remove(0),
|
||||
),
|
||||
);
|
||||
}
|
||||
for field in fields(&candidate["actual_tags"]) {
|
||||
let text = field["text"].as_str().context("Text missing")?;
|
||||
metadata.push((json!({"asset":id,"kind":"metadata","variant":candidate["variant"],"field":field["field"],"path":path,"text":text}),model.embed_documents(&[text],"File metadata")?.remove(0)));
|
||||
}
|
||||
println!("Prepared {} {}", id, candidate["variant"]);
|
||||
}
|
||||
let encoding_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut rows = Vec::new();
|
||||
for query in fixture["queries"].as_array().context("Queries missing")? {
|
||||
let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?;
|
||||
let mut lanes = Vec::new();
|
||||
for variant in [
|
||||
"audio_only",
|
||||
"original",
|
||||
"missing_metadata",
|
||||
"misleading_metadata",
|
||||
] {
|
||||
let mut ranked: Vec<_> = media
|
||||
.values()
|
||||
.map(|(source, encoded)| Ok(json!({"source":source,"score":dot(&vector,encoded)?})))
|
||||
.collect::<Result<_>>()?;
|
||||
for (source, encoded) in &metadata {
|
||||
if variant != "audio_only"
|
||||
&& (source["variant"] == variant || source["variant"] == "fixed")
|
||||
{
|
||||
let score = dot(&vector, encoded)?;
|
||||
let hit = ranked
|
||||
.iter_mut()
|
||||
.find(|h| h["source"]["asset"] == source["asset"])
|
||||
.context("Missing candidate")?;
|
||||
if score > hit["score"].as_f64().context("Score missing")? {
|
||||
*hit = json!({"source":source,"score":score});
|
||||
}
|
||||
}
|
||||
}
|
||||
ranked.sort_by(|a, b| {
|
||||
b["score"]
|
||||
.as_f64()
|
||||
.unwrap()
|
||||
.total_cmp(&a["score"].as_f64().unwrap())
|
||||
});
|
||||
lanes.push(json!({"variant":variant,"file_rankings":ranked}));
|
||||
}
|
||||
rows.push(json!({"query":query,"lanes":lanes}));
|
||||
}
|
||||
for candidate in candidates {
|
||||
ensure!(
|
||||
candidate["sha256"] == sha(&fs::read(candidate["path"].as_str().unwrap())?),
|
||||
"Candidate changed during inference"
|
||||
);
|
||||
}
|
||||
let report = json!({"fixture_sha256":sha(&fixture_raw),"prepared_sha256":sha(&plan_raw),"worker_sha256":sha(&fs::read(&args[3])?),"worker":model.info,"encoding_ms":encoding_ms,"media_inference_calls":media.len(),"metadata_inference_calls":metadata.len(),"rows":rows,"policy":"File max, real field tags,neutral File metadata title,total2048bytes,no cutoff/weights; all fixed distractors retained; first <=30s audio clips; images280","production_policy_changed":false});
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(output)?;
|
||||
file.write_all(&serde_json::to_vec_pretty(&report)?)?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn missing_and_encoder_tags_add_nothing() {
|
||||
assert!(fields(&json!({"encoder":"violin","artist":" "})).is_empty());
|
||||
}
|
||||
#[test]
|
||||
fn misleading_tags_are_literal() {
|
||||
assert_eq!(
|
||||
fields(&json!({"title":"Wrong violin"}))[0]["text"],
|
||||
"title: Wrong violin"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn total_utf8_budget_is_bounded() {
|
||||
assert!(
|
||||
fields(&json!({"title":"ä".repeat(4000)}))
|
||||
.iter()
|
||||
.map(|f| f["text"].as_str().unwrap().len())
|
||||
.sum::<usize>()
|
||||
<= 2048
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn invalid_vectors_fail() {
|
||||
assert!(dot(&[0.0], &[0.0]).is_err());
|
||||
assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user