Verify unseen music stress inputs against production decoding

tools/media-eval/prepare_music_validation.py

@@ -4,6 +4,7 @@ import hashlib
 import json
 from pathlib import Path
 import subprocess
+import struct
 import urllib.request
 
 
@@ -23,11 +24,13 @@ def probe(path, ffprobe):
     return tags
 
 
-def pcm(path, ffmpeg):
-    data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-t", "30", "-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"])
-    if not data or len(data) > 480000 * 4 or len(data) % 4:
+def pcm(path, ffmpeg, ffprobe):
+    metadata = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-of", "json", str(path)]))
+    duration_ms = min(30000, int(float(metadata["format"]["duration"]) * 1000 + 0.5))
+    data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-ss", "0.000", "-t", f"{duration_ms / 1000:.3f}", "-i", str(path), "-af", f"aresample=16000,atrim=end_sample={duration_ms * 16},asetpts=PTS-STARTPTS", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le", "pipe:1"])
+    if not data or len(data) > 480000 * 2 or len(data) % 2:
         raise ValueError("Invalid bounded PCM")
-    return sha(data)
+    return sha(b"".join(struct.pack("<f", value[0] / 32768) for value in struct.iter_unpack("<h", data)))
 
 
 def main():
@@ -74,7 +77,7 @@ def main():
                     command.extend(["-metadata", f"{key}={tags[key]}"])
             command.append(str(path))
             execute(command)
-            pcm_hash = pcm(path, args.ffmpeg)
+            pcm_hash = pcm(path, args.ffmpeg, args.ffprobe)
             if original_pcm is None:
                 original_pcm = pcm_hash
             if pcm_hash != original_pcm:

tools/media-eval/src/bin/metadata_validation.rs

@@ -98,7 +98,7 @@ fn main() -> Result<()> {
         if candidate["kind"] == "audio" {
             let actual = probe(&tools, &path, &limits)?;
             ensure!(
-                serde_json::to_value(&actual.tags)? == candidate["actual_tags"],
+                fields(&serde_json::to_value(&actual.tags)?) == fields(&candidate["actual_tags"]),
                 "Actual tags differ"
             );
             let window = audio_windows(&actual, &limits)?
@@ -215,4 +215,54 @@ mod tests {
         assert!(dot(&[0.0], &[0.0]).is_err());
         assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err());
     }
+    #[test]
+    #[ignore = "Requires the real prepared containers and CPU FFmpeg; no model inference"]
+    fn prepared_pcm_and_tags_match_production_decoder() -> Result<()> {
+        let plan_path = std::env::var_os("SEARCHI_METADATA_VALIDATION_PLAN")
+            .context("Set SEARCHI_METADATA_VALIDATION_PLAN")?;
+        let plan: Value = serde_json::from_slice(&fs::read(plan_path)?)?;
+        let tools = FfmpegTools::new(
+            PathBuf::from(
+                std::env::var_os("SEARCHI_TEST_FFMPEG").context("Set SEARCHI_TEST_FFMPEG")?,
+            ),
+            PathBuf::from(
+                std::env::var_os("SEARCHI_TEST_FFPROBE").context("Set SEARCHI_TEST_FFPROBE")?,
+            ),
+        )?;
+        let limits = MediaLimits::default();
+        let mut families = BTreeMap::new();
+        let mut verified = 0;
+        for candidate in plan["candidates"].as_array().context("Candidates absent")? {
+            if candidate.get("pcm_sha256").is_none() {
+                continue;
+            }
+            let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?);
+            ensure!(
+                sha(&fs::read(&path)?) == candidate["sha256"],
+                "Container changed"
+            );
+            let actual = probe(&tools, &path, &limits)?;
+            ensure!(
+                fields(&serde_json::to_value(&actual.tags)?) == fields(&candidate["actual_tags"]),
+                "Tag evidence differs"
+            );
+            let window = audio_windows(&actual, &limits)?
+                .into_iter()
+                .next()
+                .context("Audio absent")?;
+            let pcm = decode_audio(&tools, &path, window, &limits)?;
+            let bytes: Vec<_> = pcm.iter().flat_map(|value| value.to_le_bytes()).collect();
+            let hash = sha(&bytes);
+            ensure!(hash == candidate["pcm_sha256"], "Production PCM differs");
+            if let Some(previous) = families.insert(candidate["id"].to_string(), hash.clone()) {
+                ensure!(previous == hash, "Variant audio differs");
+            }
+            verified += 1;
+        }
+        ensure!(
+            verified == 18 && families.len() == 6,
+            "Expected six real three-variant families"
+        );
+        Ok(())
+    }
 }