Freeze unseen music stress corpus and independently verified gated DTW experiment
This commit is contained in:
karl committed
1 parent
5ab24518b4
commit
d27be302c8
14 files changed
+1305
-6
No files matched your search
@@ -0,0 +1,104 @@
|
||||
"""Bounded source acquisition and lossless metadata stress variants; no inference."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import urllib.request
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def execute(args):
|
||||
return subprocess.run(args, check=True, capture_output=True, timeout=60).stdout
|
||||
|
||||
|
||||
def probe(path, ffprobe):
|
||||
value = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)]))
|
||||
tags = {k.lower(): v for k, v in value.get("format", {}).get("tags", {}).items()}
|
||||
for stream in value["streams"]:
|
||||
tags.update({k.lower(): v for k, v in stream.get("tags", {}).items()})
|
||||
return tags
|
||||
|
||||
|
||||
def pcm(path, ffmpeg):
|
||||
data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-t", "30", "-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"])
|
||||
if not data or len(data) > 480000 * 4 or len(data) % 4:
|
||||
raise ValueError("Invalid bounded PCM")
|
||||
return sha(data)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("fixture", type=Path)
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("ffmpeg", type=Path)
|
||||
parser.add_argument("ffprobe", type=Path)
|
||||
args = parser.parse_args()
|
||||
raw = args.fixture.read_bytes()
|
||||
fixture = json.loads(raw)
|
||||
lock = json.loads(args.fixture.with_name("labels.lock.json").read_text())
|
||||
if sha(raw) != lock["fixture_sha256"]:
|
||||
raise ValueError("Frozen fixture changed")
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
plan_path = args.output / "prepared.json"
|
||||
if plan_path.exists():
|
||||
raise ValueError("Preserve existing prepared plan; use a fresh output directory")
|
||||
sources = []
|
||||
for asset in fixture["assets"]:
|
||||
destination = args.output / (asset["id"] + "-source.ogg")
|
||||
if destination.exists():
|
||||
data = destination.read_bytes()
|
||||
else:
|
||||
request = urllib.request.Request(asset["url"] + "?download=1", headers={"User-Agent": "SearchiMediaEvaluation/1.0"})
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
data = response.read(asset["bytes"] + 1)
|
||||
if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]:
|
||||
raise ValueError("Publisher bytes changed")
|
||||
destination.write_bytes(data)
|
||||
if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]:
|
||||
raise ValueError("Existing source changed")
|
||||
sources.append({"asset": asset, "path": destination.resolve(), "sha256": sha(data), "tags": probe(destination, args.ffprobe)})
|
||||
prepared = []
|
||||
whitelist = ["artist", "title", "album", "genre", "composer"]
|
||||
for i, source in enumerate(sources):
|
||||
original_pcm = None
|
||||
for variant in fixture["variants"]:
|
||||
path = args.output / (source["asset"]["id"] + "-" + variant + ".flac")
|
||||
tags = source["tags"] if variant == "original" else (sources[(i + 1) % len(sources)]["tags"] if variant == "misleading_metadata" else {})
|
||||
command = [str(args.ffmpeg), "-v", "error", "-nostdin", "-n", "-threads", "1", "-i", str(source["path"]), "-t", "30", "-ac", "1", "-ar", "16000", "-map_metadata", "-1", "-c:a", "flac"]
|
||||
for key in whitelist:
|
||||
if key in tags:
|
||||
command.extend(["-metadata", f"{key}={tags[key]}"])
|
||||
command.append(str(path))
|
||||
execute(command)
|
||||
pcm_hash = pcm(path, args.ffmpeg)
|
||||
if original_pcm is None:
|
||||
original_pcm = pcm_hash
|
||||
if pcm_hash != original_pcm:
|
||||
raise ValueError("Metadata variant changed decoded audio")
|
||||
prepared.append({"id": source["asset"]["id"], "variant": variant, "kind": "audio", "path": str(path.resolve()), "sha256": sha(path.read_bytes()), "pcm_sha256": pcm_hash, "actual_tags": probe(path, args.ffprobe), "source_sha256": source["sha256"], "source_page": source["asset"]["source_page"], "license": source["asset"]["license"]})
|
||||
# Previously evaluated assets are distractors only, never new positive evidence.
|
||||
legacy_root = Path("fixtures/media")
|
||||
legacy_raw = (legacy_root / "fixture.json").read_bytes()
|
||||
legacy = json.loads(legacy_raw)
|
||||
legacy_lock = json.loads((legacy_root / "labels.lock.json").read_text())
|
||||
if sha(legacy_raw) != legacy_lock["fixture_sha256"]:
|
||||
raise ValueError("Legacy distractor fixture changed")
|
||||
for asset in legacy["assets"]:
|
||||
if asset["id"] not in fixture["legacy_distractors"]:
|
||||
continue
|
||||
path = (legacy_root / asset["path"]).resolve()
|
||||
if sha(path.read_bytes()) != asset["sha256"]:
|
||||
raise ValueError("Legacy distractor changed")
|
||||
prepared.append({"id": asset["id"], "variant": "fixed", "kind": "image" if asset["kind"] == "image" else "audio", "path": str(path), "sha256": asset["sha256"], "actual_tags": {} if asset["kind"] == "image" else probe(path, args.ffprobe), "legacy_distractor": True})
|
||||
result = {"fixture_sha256": sha(raw), "fixture": fixture, "candidates": prepared, "ffmpeg_sha256": sha(args.ffmpeg.read_bytes()), "ffprobe_sha256": sha(args.ffprobe.read_bytes()), "source_info": [{**s, "path": str(s["path"])} for s in sources]}
|
||||
with plan_path.open("x", encoding="utf-8") as output:
|
||||
json.dump(result, output, ensure_ascii=False, indent=2)
|
||||
print("Prepared", len(prepared), "containers; no inference. Plan SHA256", sha(plan_path.read_bytes()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Mixed-pool actual metadata stress diagnostic; frozen labels never enter encodes.
|
||||
use anyhow::{ensure, Context, Result};
|
||||
use searchi_media::{audio_windows, decode_audio, probe, FfmpegTools, MediaLimits};
|
||||
use searchi_semantic::{Device, GemmaEmbedder};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{collections::BTreeMap, fs, io::Write, path::PathBuf, time::Instant};
|
||||
fn sha(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
fn fields(tags: &Value) -> Vec<Value> {
|
||||
let mut left = 2048usize;
|
||||
let mut values = Vec::new();
|
||||
for key in ["artist", "title", "album", "genre", "composer"] {
|
||||
if let Some(value) = tags[key].as_str().filter(|s| !s.trim().is_empty()) {
|
||||
let prefix = format!("{key}: ");
|
||||
let mut end = value.len().min(left.saturating_sub(prefix.len()));
|
||||
while !value.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if end == 0 {
|
||||
break;
|
||||
}
|
||||
let text = format!("{prefix}{}", &value[..end]);
|
||||
left -= text.len();
|
||||
values.push(json!({"field":key,"text":text}));
|
||||
}
|
||||
}
|
||||
values
|
||||
}
|
||||
fn dot(a: &[f32], b: &[f32]) -> Result<f64> {
|
||||
ensure!(
|
||||
a.len() == 768 && b.len() == 768 && a.iter().chain(b).all(|x| x.is_finite()),
|
||||
"Invalid vector"
|
||||
);
|
||||
Ok(a.iter()
|
||||
.zip(b)
|
||||
.map(|(a, b)| f64::from(*a) * f64::from(*b))
|
||||
.sum())
|
||||
}
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
ensure!(
|
||||
args.len() == 7,
|
||||
"Supply FIXTURE PREPARED MODEL WORKER FFMPEG FFPROBE OUTPUT"
|
||||
);
|
||||
ensure!(
|
||||
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
||||
"CPU only"
|
||||
);
|
||||
let fixture_path = PathBuf::from(&args[0]);
|
||||
let fixture_raw = fs::read(&fixture_path)?;
|
||||
let fixture: Value = serde_json::from_slice(&fixture_raw)?;
|
||||
let lock: Value =
|
||||
serde_json::from_slice(&fs::read(fixture_path.with_file_name("labels.lock.json"))?)?;
|
||||
ensure!(
|
||||
lock["fixture_sha256"] == sha(&fixture_raw),
|
||||
"Frozen labels changed"
|
||||
);
|
||||
let plan_raw = fs::read(&args[1])?;
|
||||
let plan: Value = serde_json::from_slice(&plan_raw)?;
|
||||
let input_lock: Value =
|
||||
serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
|
||||
ensure!(
|
||||
input_lock["prepared_plan_sha256"] == sha(&plan_raw)
|
||||
&& input_lock["fixture_sha256"] == sha(&fixture_raw),
|
||||
"Prepared plan differs from pre-inference input lock"
|
||||
);
|
||||
ensure!(
|
||||
plan["fixture_sha256"] == sha(&fixture_raw) && plan["fixture"] == fixture,
|
||||
"Prepared inputs mismatch"
|
||||
);
|
||||
ensure!(
|
||||
plan["ffmpeg_sha256"] == sha(&fs::read(&args[4])?)
|
||||
&& plan["ffprobe_sha256"] == sha(&fs::read(&args[5])?),
|
||||
"Decoder changed"
|
||||
);
|
||||
let output = PathBuf::from(&args[6]);
|
||||
ensure!(!output.exists(), "Preserve existing report");
|
||||
let candidates = plan["candidates"]
|
||||
.as_array()
|
||||
.context("Candidates missing")?;
|
||||
let tools = FfmpegTools::new(PathBuf::from(&args[4]), PathBuf::from(&args[5]))?;
|
||||
let limits = MediaLimits::default();
|
||||
let mut model = GemmaEmbedder::load_with_device(&args[2], &args[3], 1, Device::Cpu)?;
|
||||
ensure!(model.info.device == "cpu", "CPU required");
|
||||
let started = Instant::now();
|
||||
let mut media: BTreeMap<String, (Value, Vec<f32>)> = BTreeMap::new();
|
||||
let mut metadata = Vec::new();
|
||||
let mut decoded = BTreeMap::new();
|
||||
for candidate in candidates {
|
||||
let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?);
|
||||
ensure!(
|
||||
candidate["sha256"] == sha(&fs::read(&path)?),
|
||||
"Candidate bytes changed"
|
||||
);
|
||||
let id = candidate["id"].as_str().context("ID missing")?.to_string();
|
||||
if candidate["kind"] == "audio" {
|
||||
let actual = probe(&tools, &path, &limits)?;
|
||||
ensure!(
|
||||
serde_json::to_value(&actual.tags)? == candidate["actual_tags"],
|
||||
"Actual tags differ"
|
||||
);
|
||||
let window = audio_windows(&actual, &limits)?
|
||||
.into_iter()
|
||||
.next()
|
||||
.context("Audio absent")?;
|
||||
let pcm = decode_audio(&tools, &path, window, &limits)?;
|
||||
let bytes: Vec<_> = pcm.iter().flat_map(|s| s.to_le_bytes()).collect();
|
||||
let hash = sha(&bytes);
|
||||
if let Some(expected) = candidate["pcm_sha256"].as_str() {
|
||||
ensure!(hash == expected, "Prepared PCM changed");
|
||||
}
|
||||
if let Some(old) = decoded.insert(id.clone(), hash.clone()) {
|
||||
ensure!(old == hash, "Variants changed decoded audio");
|
||||
}
|
||||
if !media.contains_key(&id) {
|
||||
media.insert(id.clone(),(json!({"asset":id,"kind":"audio","path":path,"start_ms":0,"duration_ms":window.duration_ms}),model.embed_audio_pcm(&pcm)?));
|
||||
}
|
||||
} else if !media.contains_key(&id) {
|
||||
media.insert(
|
||||
id.clone(),
|
||||
(
|
||||
json!({"asset":id,"kind":"image","path":path}),
|
||||
model.embed_images_with_budget(&[&path], 280)?.remove(0),
|
||||
),
|
||||
);
|
||||
}
|
||||
for field in fields(&candidate["actual_tags"]) {
|
||||
let text = field["text"].as_str().context("Text missing")?;
|
||||
metadata.push((json!({"asset":id,"kind":"metadata","variant":candidate["variant"],"field":field["field"],"path":path,"text":text}),model.embed_documents(&[text],"File metadata")?.remove(0)));
|
||||
}
|
||||
println!("Prepared {} {}", id, candidate["variant"]);
|
||||
}
|
||||
let encoding_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut rows = Vec::new();
|
||||
for query in fixture["queries"].as_array().context("Queries missing")? {
|
||||
let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?;
|
||||
let mut lanes = Vec::new();
|
||||
for variant in [
|
||||
"audio_only",
|
||||
"original",
|
||||
"missing_metadata",
|
||||
"misleading_metadata",
|
||||
] {
|
||||
let mut ranked: Vec<_> = media
|
||||
.values()
|
||||
.map(|(source, encoded)| Ok(json!({"source":source,"score":dot(&vector,encoded)?})))
|
||||
.collect::<Result<_>>()?;
|
||||
for (source, encoded) in &metadata {
|
||||
if variant != "audio_only"
|
||||
&& (source["variant"] == variant || source["variant"] == "fixed")
|
||||
{
|
||||
let score = dot(&vector, encoded)?;
|
||||
let hit = ranked
|
||||
.iter_mut()
|
||||
.find(|h| h["source"]["asset"] == source["asset"])
|
||||
.context("Missing candidate")?;
|
||||
if score > hit["score"].as_f64().context("Score missing")? {
|
||||
*hit = json!({"source":source,"score":score});
|
||||
}
|
||||
}
|
||||
}
|
||||
ranked.sort_by(|a, b| {
|
||||
b["score"]
|
||||
.as_f64()
|
||||
.unwrap()
|
||||
.total_cmp(&a["score"].as_f64().unwrap())
|
||||
});
|
||||
lanes.push(json!({"variant":variant,"file_rankings":ranked}));
|
||||
}
|
||||
rows.push(json!({"query":query,"lanes":lanes}));
|
||||
}
|
||||
for candidate in candidates {
|
||||
ensure!(
|
||||
candidate["sha256"] == sha(&fs::read(candidate["path"].as_str().unwrap())?),
|
||||
"Candidate changed during inference"
|
||||
);
|
||||
}
|
||||
let report = json!({"fixture_sha256":sha(&fixture_raw),"prepared_sha256":sha(&plan_raw),"worker_sha256":sha(&fs::read(&args[3])?),"worker":model.info,"encoding_ms":encoding_ms,"media_inference_calls":media.len(),"metadata_inference_calls":metadata.len(),"rows":rows,"policy":"File max, real field tags,neutral File metadata title,total2048bytes,no cutoff/weights; all fixed distractors retained; first <=30s audio clips; images280","production_policy_changed":false});
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(output)?;
|
||||
file.write_all(&serde_json::to_vec_pretty(&report)?)?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn missing_and_encoder_tags_add_nothing() {
|
||||
assert!(fields(&json!({"encoder":"violin","artist":" "})).is_empty());
|
||||
}
|
||||
#[test]
|
||||
fn misleading_tags_are_literal() {
|
||||
assert_eq!(
|
||||
fields(&json!({"title":"Wrong violin"}))[0]["text"],
|
||||
"title: Wrong violin"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn total_utf8_budget_is_bounded() {
|
||||
assert!(
|
||||
fields(&json!({"title":"ä".repeat(4000)}))
|
||||
.iter()
|
||||
.map(|f| f["text"].as_str().unwrap().len())
|
||||
.sum::<usize>()
|
||||
<= 2048
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn invalid_vectors_fail() {
|
||||
assert!(dot(&[0.0], &[0.0]).is_err());
|
||||
assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user