Freeze unseen music stress corpus and independently verified gated DTW experiment
This commit is contained in:
karl committed
1 parent
5ab24518b4
commit
d27be302c8
14 files changed
+1305
-6
No files matched your search
@@ -0,0 +1,104 @@
|
||||
"""Bounded source acquisition and lossless metadata stress variants; no inference."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import urllib.request
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def execute(args):
|
||||
return subprocess.run(args, check=True, capture_output=True, timeout=60).stdout
|
||||
|
||||
|
||||
def probe(path, ffprobe):
|
||||
value = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)]))
|
||||
tags = {k.lower(): v for k, v in value.get("format", {}).get("tags", {}).items()}
|
||||
for stream in value["streams"]:
|
||||
tags.update({k.lower(): v for k, v in stream.get("tags", {}).items()})
|
||||
return tags
|
||||
|
||||
|
||||
def pcm(path, ffmpeg):
|
||||
data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-t", "30", "-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"])
|
||||
if not data or len(data) > 480000 * 4 or len(data) % 4:
|
||||
raise ValueError("Invalid bounded PCM")
|
||||
return sha(data)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("fixture", type=Path)
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("ffmpeg", type=Path)
|
||||
parser.add_argument("ffprobe", type=Path)
|
||||
args = parser.parse_args()
|
||||
raw = args.fixture.read_bytes()
|
||||
fixture = json.loads(raw)
|
||||
lock = json.loads(args.fixture.with_name("labels.lock.json").read_text())
|
||||
if sha(raw) != lock["fixture_sha256"]:
|
||||
raise ValueError("Frozen fixture changed")
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
plan_path = args.output / "prepared.json"
|
||||
if plan_path.exists():
|
||||
raise ValueError("Preserve existing prepared plan; use a fresh output directory")
|
||||
sources = []
|
||||
for asset in fixture["assets"]:
|
||||
destination = args.output / (asset["id"] + "-source.ogg")
|
||||
if destination.exists():
|
||||
data = destination.read_bytes()
|
||||
else:
|
||||
request = urllib.request.Request(asset["url"] + "?download=1", headers={"User-Agent": "SearchiMediaEvaluation/1.0"})
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
data = response.read(asset["bytes"] + 1)
|
||||
if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]:
|
||||
raise ValueError("Publisher bytes changed")
|
||||
destination.write_bytes(data)
|
||||
if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]:
|
||||
raise ValueError("Existing source changed")
|
||||
sources.append({"asset": asset, "path": destination.resolve(), "sha256": sha(data), "tags": probe(destination, args.ffprobe)})
|
||||
prepared = []
|
||||
whitelist = ["artist", "title", "album", "genre", "composer"]
|
||||
for i, source in enumerate(sources):
|
||||
original_pcm = None
|
||||
for variant in fixture["variants"]:
|
||||
path = args.output / (source["asset"]["id"] + "-" + variant + ".flac")
|
||||
tags = source["tags"] if variant == "original" else (sources[(i + 1) % len(sources)]["tags"] if variant == "misleading_metadata" else {})
|
||||
command = [str(args.ffmpeg), "-v", "error", "-nostdin", "-n", "-threads", "1", "-i", str(source["path"]), "-t", "30", "-ac", "1", "-ar", "16000", "-map_metadata", "-1", "-c:a", "flac"]
|
||||
for key in whitelist:
|
||||
if key in tags:
|
||||
command.extend(["-metadata", f"{key}={tags[key]}"])
|
||||
command.append(str(path))
|
||||
execute(command)
|
||||
pcm_hash = pcm(path, args.ffmpeg)
|
||||
if original_pcm is None:
|
||||
original_pcm = pcm_hash
|
||||
if pcm_hash != original_pcm:
|
||||
raise ValueError("Metadata variant changed decoded audio")
|
||||
prepared.append({"id": source["asset"]["id"], "variant": variant, "kind": "audio", "path": str(path.resolve()), "sha256": sha(path.read_bytes()), "pcm_sha256": pcm_hash, "actual_tags": probe(path, args.ffprobe), "source_sha256": source["sha256"], "source_page": source["asset"]["source_page"], "license": source["asset"]["license"]})
|
||||
# Previously evaluated assets are distractors only, never new positive evidence.
|
||||
legacy_root = Path("fixtures/media")
|
||||
legacy_raw = (legacy_root / "fixture.json").read_bytes()
|
||||
legacy = json.loads(legacy_raw)
|
||||
legacy_lock = json.loads((legacy_root / "labels.lock.json").read_text())
|
||||
if sha(legacy_raw) != legacy_lock["fixture_sha256"]:
|
||||
raise ValueError("Legacy distractor fixture changed")
|
||||
for asset in legacy["assets"]:
|
||||
if asset["id"] not in fixture["legacy_distractors"]:
|
||||
continue
|
||||
path = (legacy_root / asset["path"]).resolve()
|
||||
if sha(path.read_bytes()) != asset["sha256"]:
|
||||
raise ValueError("Legacy distractor changed")
|
||||
prepared.append({"id": asset["id"], "variant": "fixed", "kind": "image" if asset["kind"] == "image" else "audio", "path": str(path), "sha256": asset["sha256"], "actual_tags": {} if asset["kind"] == "image" else probe(path, args.ffprobe), "legacy_distractor": True})
|
||||
result = {"fixture_sha256": sha(raw), "fixture": fixture, "candidates": prepared, "ffmpeg_sha256": sha(args.ffmpeg.read_bytes()), "ffprobe_sha256": sha(args.ffprobe.read_bytes()), "source_info": [{**s, "path": str(s["path"])} for s in sources]}
|
||||
with plan_path.open("x", encoding="utf-8") as output:
|
||||
json.dump(result, output, ensure_ascii=False, indent=2)
|
||||
print("Prepared", len(prepared), "containers; no inference. Plan SHA256", sha(plan_path.read_bytes()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Mixed-pool actual metadata stress diagnostic; frozen labels never enter encodes.
|
||||
use anyhow::{ensure, Context, Result};
|
||||
use searchi_media::{audio_windows, decode_audio, probe, FfmpegTools, MediaLimits};
|
||||
use searchi_semantic::{Device, GemmaEmbedder};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{collections::BTreeMap, fs, io::Write, path::PathBuf, time::Instant};
|
||||
fn sha(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
fn fields(tags: &Value) -> Vec<Value> {
|
||||
let mut left = 2048usize;
|
||||
let mut values = Vec::new();
|
||||
for key in ["artist", "title", "album", "genre", "composer"] {
|
||||
if let Some(value) = tags[key].as_str().filter(|s| !s.trim().is_empty()) {
|
||||
let prefix = format!("{key}: ");
|
||||
let mut end = value.len().min(left.saturating_sub(prefix.len()));
|
||||
while !value.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if end == 0 {
|
||||
break;
|
||||
}
|
||||
let text = format!("{prefix}{}", &value[..end]);
|
||||
left -= text.len();
|
||||
values.push(json!({"field":key,"text":text}));
|
||||
}
|
||||
}
|
||||
values
|
||||
}
|
||||
fn dot(a: &[f32], b: &[f32]) -> Result<f64> {
|
||||
ensure!(
|
||||
a.len() == 768 && b.len() == 768 && a.iter().chain(b).all(|x| x.is_finite()),
|
||||
"Invalid vector"
|
||||
);
|
||||
Ok(a.iter()
|
||||
.zip(b)
|
||||
.map(|(a, b)| f64::from(*a) * f64::from(*b))
|
||||
.sum())
|
||||
}
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
ensure!(
|
||||
args.len() == 7,
|
||||
"Supply FIXTURE PREPARED MODEL WORKER FFMPEG FFPROBE OUTPUT"
|
||||
);
|
||||
ensure!(
|
||||
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
||||
"CPU only"
|
||||
);
|
||||
let fixture_path = PathBuf::from(&args[0]);
|
||||
let fixture_raw = fs::read(&fixture_path)?;
|
||||
let fixture: Value = serde_json::from_slice(&fixture_raw)?;
|
||||
let lock: Value =
|
||||
serde_json::from_slice(&fs::read(fixture_path.with_file_name("labels.lock.json"))?)?;
|
||||
ensure!(
|
||||
lock["fixture_sha256"] == sha(&fixture_raw),
|
||||
"Frozen labels changed"
|
||||
);
|
||||
let plan_raw = fs::read(&args[1])?;
|
||||
let plan: Value = serde_json::from_slice(&plan_raw)?;
|
||||
let input_lock: Value =
|
||||
serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?;
|
||||
ensure!(
|
||||
input_lock["prepared_plan_sha256"] == sha(&plan_raw)
|
||||
&& input_lock["fixture_sha256"] == sha(&fixture_raw),
|
||||
"Prepared plan differs from pre-inference input lock"
|
||||
);
|
||||
ensure!(
|
||||
plan["fixture_sha256"] == sha(&fixture_raw) && plan["fixture"] == fixture,
|
||||
"Prepared inputs mismatch"
|
||||
);
|
||||
ensure!(
|
||||
plan["ffmpeg_sha256"] == sha(&fs::read(&args[4])?)
|
||||
&& plan["ffprobe_sha256"] == sha(&fs::read(&args[5])?),
|
||||
"Decoder changed"
|
||||
);
|
||||
let output = PathBuf::from(&args[6]);
|
||||
ensure!(!output.exists(), "Preserve existing report");
|
||||
let candidates = plan["candidates"]
|
||||
.as_array()
|
||||
.context("Candidates missing")?;
|
||||
let tools = FfmpegTools::new(PathBuf::from(&args[4]), PathBuf::from(&args[5]))?;
|
||||
let limits = MediaLimits::default();
|
||||
let mut model = GemmaEmbedder::load_with_device(&args[2], &args[3], 1, Device::Cpu)?;
|
||||
ensure!(model.info.device == "cpu", "CPU required");
|
||||
let started = Instant::now();
|
||||
let mut media: BTreeMap<String, (Value, Vec<f32>)> = BTreeMap::new();
|
||||
let mut metadata = Vec::new();
|
||||
let mut decoded = BTreeMap::new();
|
||||
for candidate in candidates {
|
||||
let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?);
|
||||
ensure!(
|
||||
candidate["sha256"] == sha(&fs::read(&path)?),
|
||||
"Candidate bytes changed"
|
||||
);
|
||||
let id = candidate["id"].as_str().context("ID missing")?.to_string();
|
||||
if candidate["kind"] == "audio" {
|
||||
let actual = probe(&tools, &path, &limits)?;
|
||||
ensure!(
|
||||
serde_json::to_value(&actual.tags)? == candidate["actual_tags"],
|
||||
"Actual tags differ"
|
||||
);
|
||||
let window = audio_windows(&actual, &limits)?
|
||||
.into_iter()
|
||||
.next()
|
||||
.context("Audio absent")?;
|
||||
let pcm = decode_audio(&tools, &path, window, &limits)?;
|
||||
let bytes: Vec<_> = pcm.iter().flat_map(|s| s.to_le_bytes()).collect();
|
||||
let hash = sha(&bytes);
|
||||
if let Some(expected) = candidate["pcm_sha256"].as_str() {
|
||||
ensure!(hash == expected, "Prepared PCM changed");
|
||||
}
|
||||
if let Some(old) = decoded.insert(id.clone(), hash.clone()) {
|
||||
ensure!(old == hash, "Variants changed decoded audio");
|
||||
}
|
||||
if !media.contains_key(&id) {
|
||||
media.insert(id.clone(),(json!({"asset":id,"kind":"audio","path":path,"start_ms":0,"duration_ms":window.duration_ms}),model.embed_audio_pcm(&pcm)?));
|
||||
}
|
||||
} else if !media.contains_key(&id) {
|
||||
media.insert(
|
||||
id.clone(),
|
||||
(
|
||||
json!({"asset":id,"kind":"image","path":path}),
|
||||
model.embed_images_with_budget(&[&path], 280)?.remove(0),
|
||||
),
|
||||
);
|
||||
}
|
||||
for field in fields(&candidate["actual_tags"]) {
|
||||
let text = field["text"].as_str().context("Text missing")?;
|
||||
metadata.push((json!({"asset":id,"kind":"metadata","variant":candidate["variant"],"field":field["field"],"path":path,"text":text}),model.embed_documents(&[text],"File metadata")?.remove(0)));
|
||||
}
|
||||
println!("Prepared {} {}", id, candidate["variant"]);
|
||||
}
|
||||
let encoding_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut rows = Vec::new();
|
||||
for query in fixture["queries"].as_array().context("Queries missing")? {
|
||||
let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?;
|
||||
let mut lanes = Vec::new();
|
||||
for variant in [
|
||||
"audio_only",
|
||||
"original",
|
||||
"missing_metadata",
|
||||
"misleading_metadata",
|
||||
] {
|
||||
let mut ranked: Vec<_> = media
|
||||
.values()
|
||||
.map(|(source, encoded)| Ok(json!({"source":source,"score":dot(&vector,encoded)?})))
|
||||
.collect::<Result<_>>()?;
|
||||
for (source, encoded) in &metadata {
|
||||
if variant != "audio_only"
|
||||
&& (source["variant"] == variant || source["variant"] == "fixed")
|
||||
{
|
||||
let score = dot(&vector, encoded)?;
|
||||
let hit = ranked
|
||||
.iter_mut()
|
||||
.find(|h| h["source"]["asset"] == source["asset"])
|
||||
.context("Missing candidate")?;
|
||||
if score > hit["score"].as_f64().context("Score missing")? {
|
||||
*hit = json!({"source":source,"score":score});
|
||||
}
|
||||
}
|
||||
}
|
||||
ranked.sort_by(|a, b| {
|
||||
b["score"]
|
||||
.as_f64()
|
||||
.unwrap()
|
||||
.total_cmp(&a["score"].as_f64().unwrap())
|
||||
});
|
||||
lanes.push(json!({"variant":variant,"file_rankings":ranked}));
|
||||
}
|
||||
rows.push(json!({"query":query,"lanes":lanes}));
|
||||
}
|
||||
for candidate in candidates {
|
||||
ensure!(
|
||||
candidate["sha256"] == sha(&fs::read(candidate["path"].as_str().unwrap())?),
|
||||
"Candidate changed during inference"
|
||||
);
|
||||
}
|
||||
let report = json!({"fixture_sha256":sha(&fixture_raw),"prepared_sha256":sha(&plan_raw),"worker_sha256":sha(&fs::read(&args[3])?),"worker":model.info,"encoding_ms":encoding_ms,"media_inference_calls":media.len(),"metadata_inference_calls":metadata.len(),"rows":rows,"policy":"File max, real field tags,neutral File metadata title,total2048bytes,no cutoff/weights; all fixed distractors retained; first <=30s audio clips; images280","production_policy_changed":false});
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(output)?;
|
||||
file.write_all(&serde_json::to_vec_pretty(&report)?)?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn missing_and_encoder_tags_add_nothing() {
|
||||
assert!(fields(&json!({"encoder":"violin","artist":" "})).is_empty());
|
||||
}
|
||||
#[test]
|
||||
fn misleading_tags_are_literal() {
|
||||
assert_eq!(
|
||||
fields(&json!({"title":"Wrong violin"}))[0]["text"],
|
||||
"title: Wrong violin"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn total_utf8_budget_is_bounded() {
|
||||
assert!(
|
||||
fields(&json!({"title":"ä".repeat(4000)}))
|
||||
.iter()
|
||||
.map(|f| f["text"].as_str().unwrap().len())
|
||||
.sum::<usize>()
|
||||
<= 2048
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn invalid_vectors_fail() {
|
||||
assert!(dot(&[0.0], &[0.0]).is_err());
|
||||
assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err());
|
||||
}
|
||||
}
|
||||
@@ -83,3 +83,13 @@ in the embedded ASR evidence and are independently checked; they do not authoriz
|
||||
fabricating or clamping DTW points. Missing/invalid DTW points exclude a window
|
||||
and prevent accepted scoring. Use a separate DTW report/output and retain the
|
||||
failed ordinary-v1 run. The frozen five sources/eight queries remain unchanged.
|
||||
|
||||
The separately frozen `gated-dtw-v1` combination embeds exact manifest bytes
|
||||
with SHA `d3bf453f21dac03fd029344af2fe700480cf686d2ba5b5d7fcc79134dda1c563`.
|
||||
The ranker verifies component hashes and the original frozen VAD bytes, rebuilds
|
||||
all twelve speech windows from source intersections, requires explicit no-speech
|
||||
status for empty competitors, then independently reconstructs DTW points. All
|
||||
five sources remain mandatory; dropped windows, fabricated no-speech claims,
|
||||
tampered policy bytes and invalid points are rejected. This combination leaves
|
||||
the old gated base and ungated DTW policies unchanged. Structural token tests
|
||||
provide no new quality result; actual model inference remains a separate gate.
|
||||
@@ -12,8 +12,50 @@ pub const DTW_POLICY: &str =
|
||||
pub const DTW_POLICY_SHA: &str = "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5";
|
||||
pub const DTW_CONTRACT_SHA: &str =
|
||||
"61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f";
|
||||
pub const COMBINATION_POLICY: &str =
|
||||
"gated-dtw-v1;frozen-silero-source-intersections;dtw-token-sentences-v1;all-five-sources";
|
||||
pub const COMBINATION_SHA: &str =
|
||||
"d3bf453f21dac03fd029344af2fe700480cf686d2ba5b5d7fcc79134dda1c563";
|
||||
|
||||
fn verify_combination(asr: &Value) -> Result<()> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let bytes = asr["combination_policy_utf8"]
|
||||
.as_str()
|
||||
.context("Missing original combination policy")?
|
||||
.as_bytes();
|
||||
ensure!(
|
||||
bytes.len() <= 16384
|
||||
&& format!("{:x}", Sha256::digest(bytes)) == COMBINATION_SHA
|
||||
&& asr["combination_policy_sha256"] == COMBINATION_SHA
|
||||
&& asr["combination_policy"] == COMBINATION_POLICY,
|
||||
"Frozen combined policy changed"
|
||||
);
|
||||
let plan: Value = serde_json::from_slice(bytes)?;
|
||||
ensure!(
|
||||
plan["policy"] == COMBINATION_POLICY
|
||||
&& plan["dtw_policy_sha256"] == DTW_POLICY_SHA
|
||||
&& plan["dtw_contract_sha256"] == DTW_CONTRACT_SHA
|
||||
&& plan["ordinary_contract_sha256"] == CONTRACT_SHA
|
||||
&& plan["asr_worker_sha256"] == asr["worker_sha256"]
|
||||
&& plan["vad_report_sha256"] == asr["vad_report_sha256"]
|
||||
&& plan["sources"] == json!(super::ASSETS),
|
||||
"Combined component provenance differs"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn enabled(asr: &Value) -> Result<bool> {
|
||||
let combined = asr["policy"] == super::gated::POLICY && asr["alignment_policy"] == DTW_POLICY;
|
||||
if combined {
|
||||
verify_combination(asr)?;
|
||||
} else {
|
||||
ensure!(
|
||||
asr.get("combination_policy").is_none()
|
||||
&& asr.get("combination_policy_sha256").is_none()
|
||||
&& asr.get("combination_policy_utf8").is_none(),
|
||||
"Combined policy outside declared gated DTW mode"
|
||||
);
|
||||
}
|
||||
if asr.get("alignment_policy").is_none() {
|
||||
ensure!(
|
||||
asr.get("alignment_contract_sha256").is_none()
|
||||
@@ -32,7 +74,7 @@ pub fn enabled(asr: &Value) -> Result<bool> {
|
||||
&& asr["candidate"] == "base"
|
||||
&& asr["worker_sha256"]
|
||||
== "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8"
|
||||
&& asr["policy"] == super::POLICY,
|
||||
&& (asr["policy"] == super::POLICY || combined),
|
||||
"Unknown token alignment provenance"
|
||||
);
|
||||
Ok(true)
|
||||
|
||||
@@ -374,6 +374,10 @@ fn main() -> Result<()> {
|
||||
report["alignment_policy"] = asr["alignment_policy"].clone();
|
||||
report["alignment_policy_sha256"] = asr["alignment_policy_sha256"].clone();
|
||||
report["alignment_contract_sha256"] = asr["alignment_contract_sha256"].clone();
|
||||
if asr.get("combination_policy").is_some() {
|
||||
report["combination_policy"] = asr["combination_policy"].clone();
|
||||
report["combination_policy_sha256"] = asr["combination_policy_sha256"].clone();
|
||||
}
|
||||
}
|
||||
report["phase"] = json!("document_encoding");
|
||||
checkpoint(&output, &report)?;
|
||||
@@ -487,6 +491,68 @@ mod tests {
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_gated_dtw_pool_requires_all_sources_windows_and_policy_bytes() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let path = root.join(".searchi/speech/gated-base-results.json");
|
||||
if !path.exists() {
|
||||
return;
|
||||
}
|
||||
let fixture: Value =
|
||||
serde_json::from_slice(&fs::read(root.join("fixtures/media/fixture.json")).unwrap())
|
||||
.unwrap();
|
||||
let mut report: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap();
|
||||
let plan = include_str!("../../speech-vad/gated-dtw-policy.json");
|
||||
report["combination_policy_utf8"] = json!(plan);
|
||||
report["combination_policy"] = json!(alignment::COMBINATION_POLICY);
|
||||
report["combination_policy_sha256"] = json!(alignment::COMBINATION_SHA);
|
||||
report["alignment_policy"] = json!(alignment::DTW_POLICY);
|
||||
report["alignment_policy_sha256"] = json!(alignment::DTW_POLICY_SHA);
|
||||
report["alignment_contract_sha256"] = json!(alignment::DTW_CONTRACT_SHA);
|
||||
// Explicitly synthetic token points test the coverage/protocol path,
|
||||
// not actual alignment or quality. Original frozen VAD windows unchanged.
|
||||
for row in report["rows"].as_array_mut().unwrap() {
|
||||
for window in row["windows"].as_array_mut().unwrap() {
|
||||
for segment in window["actual_whisper_json"]["transcription"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
{
|
||||
segment["tokens"] = json!([{"id":100,"text":segment["text"],"offsets":segment["offsets"],"t_dtw":number(&segment["offsets"]["from"]).unwrap()/10}]);
|
||||
}
|
||||
window["aligned_segments"] = json!(alignment::reconstruct_mode(
|
||||
&window["actual_whisper_json"],
|
||||
number(&window["offset_ms"]).unwrap(),
|
||||
number(&window["duration_ms"]).unwrap(),
|
||||
true
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
}
|
||||
assert!(eligible(&report, &fixture).unwrap().1.is_empty());
|
||||
let mut changed = report.clone();
|
||||
changed["combination_policy_utf8"] = json!(format!("{plan}\n"));
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report.clone();
|
||||
changed["rows"].as_array_mut().unwrap().pop();
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report.clone();
|
||||
changed["rows"][0]["windows"].as_array_mut().unwrap().pop();
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report.clone();
|
||||
let no_speech = changed["rows"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.iter_mut()
|
||||
.find(|row| row["status"] == "no_speech")
|
||||
.unwrap();
|
||||
no_speech["asr_not_run"] = json!(false);
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report;
|
||||
changed["rows"][0]["windows"][0]["actual_whisper_json"]["transcription"][0]["tokens"][0]
|
||||
["t_dtw"] = json!(-1);
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aligned_pool_preserves_all_competitors_and_marks_real_failures_ineligible() {
|
||||
let hash = "a".repeat(64);
|
||||
|
||||
@@ -36,3 +36,36 @@ VAD report. It preserves all five competitors, no-speech status, original source
|
||||
intervals and raw ASR logs. `compare.py BASELINE GATED` reports component timing
|
||||
and segment counts only; it rejects incomplete or invalid pools and does not
|
||||
score retrieval quality. See the protocol document for actual evidence.
|
||||
# Separate frozen VAD + DTW comparison
|
||||
|
||||
`gated-dtw-policy.json` combines the already frozen Silero source-intersection
|
||||
decisions with the separately declared base DTW sentence-point contract.
|
||||
Add `--dtw-sentences` to `gated_asr.py` and select a new output, for example
|
||||
`.searchi/speech/gated-base-dtw-results-v1.json`. All five source rows remain
|
||||
mandatory; explicit no-speech rows use unchanged actual frozen VAD evidence and
|
||||
launch no ASR. Neither relevance labels nor the observed malformed piano output
|
||||
selects a gate or a window. Original raw Google audio retrieval stays unchanged.
|
||||
Original raw recognized segments and ordinary token errors remain recorded.
|
||||
Valid DTW points alone determine the separate aligned candidate times; missing
|
||||
or invalid points make accepted comparison scoring ineligible.
|
||||
|
||||
The combined manifest pins unchanged VAD report/worker, base worker, DTW policy,
|
||||
DTW helper and ordinary helper hashes. It preserves the old gated policy in
|
||||
`policy`, with separate `alignment_policy` and `combination_policy` provenance.
|
||||
Rankers must independently validate both policies and reconstruct every frozen
|
||||
VAD window and DTW point before inference/scoring. Existing gated base output
|
||||
is not overwritten. A combination plan or structural tests establish no new
|
||||
speech recognition or timestamp quality.
|
||||
|
||||
Run only after the coordinator freezes the plan and grants the exclusive CPU
|
||||
slot. Use `--no-gpu --threads 1 --processors 1 --dtw base --no-flash-attn`; the
|
||||
harness adds these explicitly. The twelve existing speech windows remain
|
||||
bounded to 30 seconds, 4 MiB raw output and 600 seconds each. Observed gated
|
||||
base processing previously took about 65 seconds including VAD cost; DTW cost
|
||||
and memory have not been measured. Native DTW workspace defaults to an additional
|
||||
128 MiB, which is not a total memory ceiling. Strict parse failures retain bounded
|
||||
raw bytes and atomically persist incomplete status, then stop instead of skipping
|
||||
a competitor. No full rerun has established this combined policy.
|
||||
DTW native logs are retained byte-exact as bounded base64 plus SHA so malformed
|
||||
recognized bytes in diagnostic stderr cannot cause lossy decoding. Recognized
|
||||
JSON remains strict UTF-8 and cannot be repaired by this log representation.
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"policy": "gated-dtw-v1;frozen-silero-source-intersections;dtw-token-sentences-v1;all-five-sources",
|
||||
"vad_report_sha256": "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df",
|
||||
"vad_worker_sha256": "28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19",
|
||||
"asr_worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8",
|
||||
"dtw_policy_sha256": "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5",
|
||||
"dtw_contract_sha256": "61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f",
|
||||
"ordinary_contract_sha256": "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520",
|
||||
"sources": ["clip04", "music01", "music02", "sound01", "sound02"],
|
||||
"rules": "Unchanged frozen VAD decisions/windows; explicit no-speech rows launch no ASR; retained original raw audio retrieval is unchanged; no labels select speech intervals or suppress sources",
|
||||
"production_default": false
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import sys
|
||||
import base64
|
||||
|
||||
spec = importlib.util.spec_from_file_location("gated_speech_contract", Path(__file__).parents[1] / "speech-eval/run.py")
|
||||
speech = importlib.util.module_from_spec(spec)
|
||||
@@ -16,6 +18,22 @@ VAD_SHA = "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df"
|
||||
VAD_WORKER_SHA = "28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19"
|
||||
WHISPER_SHA = "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8"
|
||||
POLICY = "gated-vad-v1;source-intersections;integer-ms-tail-remainder-recorded;contiguous30s;independent-process;auto-language;segment-times"
|
||||
HARNESS_SHA_AT_START = speech.sha256(Path(__file__))
|
||||
|
||||
|
||||
def dtw_contract():
|
||||
plan_path = Path(__file__).with_name("gated-dtw-policy.json")
|
||||
plan = json.loads(plan_path.read_text(encoding="utf-8"))
|
||||
directory = Path(__file__).parents[1] / "speech-eval"
|
||||
for filename, key in [("dtw-policy.json", "dtw_policy_sha256"),
|
||||
("dtw_alignment.py", "dtw_contract_sha256"),
|
||||
("token_alignment.py", "ordinary_contract_sha256")]:
|
||||
if speech.sha256(directory / filename) != plan[key]:
|
||||
raise ValueError("Frozen gated DTW contract mismatch")
|
||||
if (plan["vad_report_sha256"] != VAD_SHA or plan["asr_worker_sha256"] != WHISPER_SHA
|
||||
or plan["vad_worker_sha256"] != VAD_WORKER_SHA or plan["sources"] != list(speech.ASSETS)):
|
||||
raise ValueError("Frozen gated DTW provenance mismatch")
|
||||
return plan, speech.sha256(plan_path)
|
||||
|
||||
|
||||
def interval_windows(intervals, samples, origin):
|
||||
@@ -72,7 +90,13 @@ def main():
|
||||
for option in ("whisper", "model", "ffmpeg", "ffprobe", "vad-report", "scratch-parent", "output"):
|
||||
parser.add_argument("--" + option, type=Path, required=True)
|
||||
parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
|
||||
parser.add_argument("--dtw-sentences", action="store_true", help="Separate frozen VAD+DTW comparison")
|
||||
args = parser.parse_args()
|
||||
if args.dtw_sentences:
|
||||
combined, combined_sha = dtw_contract()
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "speech-eval"))
|
||||
from dtw_alignment import aligned_segments, POLICY as DTW_POLICY
|
||||
from token_alignment import aligned_segments as ordinary_segments
|
||||
if args.output.exists():
|
||||
raise ValueError("Never overwrite experiment evidence")
|
||||
if speech.sha256(args.vad_report) != VAD_SHA or speech.sha256(args.fixture) != speech.FIXTURE_SHA:
|
||||
@@ -96,17 +120,23 @@ def main():
|
||||
"worker_sha256": WHISPER_SHA, "backend": "forced CPU", "threads": 1, "policy": POLICY,
|
||||
"vad_report_sha256": VAD_SHA, "actual_vad_report": document,
|
||||
"vad_report_utf8": vad_bytes.decode("utf-8"),
|
||||
"harness_sha256": speech.sha256(Path(__file__)),
|
||||
"harness_sha256": HARNESS_SHA_AT_START,
|
||||
"speech_contract_sha256": speech.sha256(Path(spec.origin)),
|
||||
"ffmpeg_sha256": document["ffmpeg_sha256"], "ffprobe_sha256": document["ffprobe_sha256"],
|
||||
"scope": "Gated ASR diagnostic; no product or word-alignment claim", "rows": []}
|
||||
if args.dtw_sentences:
|
||||
report.update(alignment_policy=DTW_POLICY,
|
||||
alignment_policy_sha256=combined["dtw_policy_sha256"],
|
||||
alignment_contract_sha256=combined["dtw_contract_sha256"],
|
||||
combination_policy=combined["policy"], combination_policy_sha256=combined_sha,
|
||||
combination_policy_utf8=Path(__file__).with_name("gated-dtw-policy.json").read_bytes().decode("utf-8"))
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.scratch_parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("x", encoding="utf-8") as stream:
|
||||
json.dump(report, stream)
|
||||
|
||||
def checkpoint():
|
||||
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
speech.checkpoint_report(args.output, report)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gated-asr-", dir=args.scratch_parent) as directory:
|
||||
scratch = Path(directory)
|
||||
@@ -137,15 +167,44 @@ def main():
|
||||
frames = speech.decoded_frames(wav, length)
|
||||
asr_ms = speech.run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
|
||||
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
|
||||
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, speech.MAX_OUTPUT)], 600)
|
||||
raw = json.loads(output.read_text(encoding="utf-8"))
|
||||
"--output-json-full" if args.dtw_sentences else "--output-json",
|
||||
*(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []),
|
||||
"--output-file", str(prefix)], scratch, "asr", [(output, speech.MAX_OUTPUT)], 600)
|
||||
try:
|
||||
raw = speech.parse_asr_output(output, args.scratch_parent)
|
||||
except speech.AsrOutputError as error:
|
||||
row["windows"].append({**planned, "decoded_frames": frames,
|
||||
"error_stage": "asr_output_parse", "validation_error": str(error),
|
||||
"actual_output_failure": error.failure, "segments": []})
|
||||
report.update(quality_failed=True, comparison_eligible=False,
|
||||
failure={"asset":asset["id"], "offset_ms":offset, **error.failure})
|
||||
checkpoint()
|
||||
raise
|
||||
window = speech.recognized_window(raw, offset, length, decode_ms, asr_ms)
|
||||
window.update(planned)
|
||||
window["decoded_frames"] = frames
|
||||
window["actual_asr_log"] = (scratch / "asr.log").read_text(encoding="utf-8")
|
||||
if args.dtw_sentences:
|
||||
log_bytes = (scratch / "asr.log").read_bytes()
|
||||
if len(log_bytes) > speech.MAX_OUTPUT:
|
||||
raise ValueError("ASR log exceeds bound")
|
||||
window["actual_asr_log_base64"] = base64.b64encode(log_bytes).decode("ascii")
|
||||
window["actual_asr_log_sha256"] = speech.sha256(scratch / "asr.log")
|
||||
else:
|
||||
window["actual_asr_log"] = (scratch / "asr.log").read_text(encoding="utf-8")
|
||||
if args.dtw_sentences:
|
||||
try:
|
||||
ordinary_segments(raw, offset, length)
|
||||
except (ValueError, KeyError, TypeError) as error:
|
||||
window["ordinary_alignment_validation_error"] = str(error)
|
||||
try:
|
||||
window["aligned_segments"] = aligned_segments(raw, offset, length)
|
||||
except (ValueError, KeyError, TypeError) as error:
|
||||
window["alignment_validation_error"] = str(error)
|
||||
row["windows"].append(window)
|
||||
if "validation_error" in window:
|
||||
report["validation_failures"].append({"asset": asset["id"], "offset_ms": offset, "error": window["validation_error"]})
|
||||
if "alignment_validation_error" in window:
|
||||
report["validation_failures"].append({"asset": asset["id"], "offset_ms": offset, "error": window["alignment_validation_error"]})
|
||||
checkpoint()
|
||||
if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]:
|
||||
raise ValueError("Source changed during recognition")
|
||||
@@ -161,6 +220,8 @@ def main():
|
||||
or any(speech.sha256(getattr(args, name)) != document[name + "_sha256"] for name in ("ffmpeg", "ffprobe"))):
|
||||
raise ValueError("Experiment tools/provenance changed during recognition")
|
||||
report["complete"] = True
|
||||
if args.dtw_sentences and dtw_contract()[1] != combined_sha:
|
||||
raise ValueError("Gated DTW contracts changed during recognition")
|
||||
report["quality_failed"] = bool(report["validation_failures"])
|
||||
report["comparison_eligible"] = not report["quality_failed"]
|
||||
checkpoint()
|
||||
|
||||
@@ -3,11 +3,20 @@ import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import gated_asr as gated
|
||||
|
||||
|
||||
class GatedPlanning(unittest.TestCase):
|
||||
def test_frozen_dtw_combination_preserves_sources_and_component_hashes(self):
|
||||
plan, digest = gated.dtw_contract()
|
||||
self.assertEqual(plan["sources"], list(gated.speech.ASSETS))
|
||||
self.assertEqual(plan["vad_report_sha256"], gated.VAD_SHA)
|
||||
self.assertEqual(len(digest), 64)
|
||||
with patch.object(gated.speech, "sha256", return_value="changed"):
|
||||
with self.assertRaises(ValueError):
|
||||
gated.dtw_contract()
|
||||
def test_absolute_offset_and_contiguous_split(self):
|
||||
interval = gated.vad.source_intersection([10, 6110], 1000000, 5125)
|
||||
windows = gated.interval_windows([interval], 1000000, 5125)
|
||||
|
||||
Reference in new issue
Block a user