Verify unseen music stress inputs against production decoding

fixtures/music-validation-v1/README.md

@@ -55,18 +55,25 @@ metadata variants share recordings and are correlated, not independent examples.
 ## Run after reserving the CPU slot
 
 ```powershell
-python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1 .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe
+python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1-canonical .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe
 cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked
 $env:SEARCHI_FORCE_CPU = '1'
-cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1/results.json
+$env:SEARCHI_METADATA_VALIDATION_PLAN = 'C:/Users/karl/searchi/.searchi/qa/music-validation-v1-canonical/prepared.json'
+$env:SEARCHI_TEST_FFMPEG = 'C:/Users/karl/searchi/.searchi/media-runtime/bin/ffmpeg.exe'
+$env:SEARCHI_TEST_FFPROBE = 'C:/Users/karl/searchi/.searchi/media-runtime/bin/ffprobe.exe'
+cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- --include-ignored
+cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1-canonical/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1-canonical/results.json
 ```
 
 Preparation already completed without inference. Its frozen plan SHA-256 is
-`8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2`;
+`3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78`;
 fixture SHA-256 is
 `f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577`.
 Use a fresh output directory for another preparation; existing plans and result
 files are preserved. Decoder hashes and actual container tags are checked.
+Five tests passed, including actual production decoder/probe parity on all
+18 stress containers; strict Clippy passed. This preflight performed no model
+inference. Adjust the local fixture/helper paths for another development machine.
 
 The diagnostic uses actual Google EmbeddingGemma 2 on CPU, one thread. Each of
 the eight distinct audio sources is encoded once and both photos at 280 tokens.
@@ -76,3 +83,5 @@ citations remain explicit; a metadata hit has no fabricated timestamp. Four
 lanes compare media-only, original tags, missing tags and misleading tags without
 tuned weights or cutoff. Report positive recall and no-answer failure separately.
 This direct cosine diagnostic is not production ANN/hybrid validation.
+
+The earlier prepared plan 8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2 was never inferred. Pre-inference review found that its verification decoded full float PCM while the production decoder uses millisecond trimming and s16 normalization. The corrected plan above matches the actual production decoder and compares only the whitelisted tag evidence; labels, source bytes, query texts and ranking hypothesis are unchanged. The earlier plan is preserved under its original ignored directory.

fixtures/music-validation-v1/inputs.lock.json

@@ -1,7 +1,7 @@
 {
   "frozen_before_inference": true,
   "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577",
-  "prepared_plan_sha256": "8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2",
+  "prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78",
   "sources": [
     {
       "id": "unseen01",
@@ -80,7 +80,7 @@
         "artist": "Ryan",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23"
+      "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
     },
     {
       "id": "unseen01",
@@ -90,7 +90,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23"
+      "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
     },
     {
       "id": "unseen01",
@@ -101,7 +101,7 @@
         "title": "cello strings",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23"
+      "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
     },
     {
       "id": "unseen02",
@@ -112,7 +112,7 @@
         "title": "cello strings",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed"
+      "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
     },
     {
       "id": "unseen02",
@@ -122,7 +122,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed"
+      "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
     },
     {
       "id": "unseen02",
@@ -132,7 +132,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed"
+      "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
     },
     {
       "id": "unseen03",
@@ -142,7 +142,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504"
+      "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
     },
     {
       "id": "unseen03",
@@ -152,7 +152,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504"
+      "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
     },
     {
       "id": "unseen03",
@@ -164,7 +164,7 @@
         "title": "Bucket mute",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504"
+      "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
     },
     {
       "id": "unseen04",
@@ -176,7 +176,7 @@
         "title": "Bucket mute",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161"
+      "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
     },
     {
       "id": "unseen04",
@@ -186,7 +186,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161"
+      "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
     },
     {
       "id": "unseen04",
@@ -198,7 +198,7 @@
         "title": "After you've gone - first 8 bars on tenor sax",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161"
+      "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
     },
     {
       "id": "unseen05",
@@ -210,7 +210,7 @@
         "title": "After you've gone - first 8 bars on tenor sax",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd"
+      "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
     },
     {
       "id": "unseen05",
@@ -220,7 +220,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd"
+      "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
     },
     {
       "id": "unseen05",
@@ -232,7 +232,7 @@
         "title": "Ain't She Sweet, first 8 bars on soprano sax",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd"
+      "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
     },
     {
       "id": "unseen06",
@@ -244,7 +244,7 @@
         "title": "Ain't She Sweet, first 8 bars on soprano sax",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b"
+      "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
     },
     {
       "id": "unseen06",
@@ -254,7 +254,7 @@
       "actual_tags": {
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b"
+      "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
     },
     {
       "id": "unseen06",
@@ -265,7 +265,7 @@
         "artist": "Ryan",
         "encoder": "Lavf63.1.102"
       },
-      "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b"
+      "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
     },
     {
       "id": "image01",

tools/media-eval/prepare_music_validation.py

@@ -4,6 +4,7 @@ import hashlib
 import json
 from pathlib import Path
 import subprocess
+import struct
 import urllib.request
 
 
@@ -23,11 +24,13 @@ def probe(path, ffprobe):
     return tags
 
 
-def pcm(path, ffmpeg):
+def pcm(path, ffmpeg, ffprobe):
-    data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-t", "30", "-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"])
+    metadata = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-of", "json", str(path)]))
-    if not data or len(data) > 480000 * 4 or len(data) % 4:
+    duration_ms = min(30000, int(float(metadata["format"]["duration"]) * 1000 + 0.5))
+    data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-ss", "0.000", "-t", f"{duration_ms / 1000:.3f}", "-i", str(path), "-af", f"aresample=16000,atrim=end_sample={duration_ms * 16},asetpts=PTS-STARTPTS", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le", "pipe:1"])
+    if not data or len(data) > 480000 * 2 or len(data) % 2:
         raise ValueError("Invalid bounded PCM")
-    return sha(data)
+    return sha(b"".join(struct.pack("<f", value[0] / 32768) for value in struct.iter_unpack("<h", data)))
 
 
 def main():
@@ -74,7 +77,7 @@ def main():
                     command.extend(["-metadata", f"{key}={tags[key]}"])
             command.append(str(path))
             execute(command)
-            pcm_hash = pcm(path, args.ffmpeg)
+            pcm_hash = pcm(path, args.ffmpeg, args.ffprobe)
             if original_pcm is None:
                 original_pcm = pcm_hash
             if pcm_hash != original_pcm:

tools/media-eval/src/bin/metadata_validation.rs

@@ -98,7 +98,7 @@ fn main() -> Result<()> {
         if candidate["kind"] == "audio" {
             let actual = probe(&tools, &path, &limits)?;
             ensure!(
-                serde_json::to_value(&actual.tags)? == candidate["actual_tags"],
+                fields(&serde_json::to_value(&actual.tags)?) == fields(&candidate["actual_tags"]),
                 "Actual tags differ"
             );
             let window = audio_windows(&actual, &limits)?
@@ -215,4 +215,54 @@ mod tests {
         assert!(dot(&[0.0], &[0.0]).is_err());
         assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err());
     }
+    #[test]
+    #[ignore = "Requires the real prepared containers and CPU FFmpeg; no model inference"]
+    fn prepared_pcm_and_tags_match_production_decoder() -> Result<()> {
+        let plan_path = std::env::var_os("SEARCHI_METADATA_VALIDATION_PLAN")
+            .context("Set SEARCHI_METADATA_VALIDATION_PLAN")?;
+        let plan: Value = serde_json::from_slice(&fs::read(plan_path)?)?;
+        let tools = FfmpegTools::new(
+            PathBuf::from(
+                std::env::var_os("SEARCHI_TEST_FFMPEG").context("Set SEARCHI_TEST_FFMPEG")?,
+            ),
+            PathBuf::from(
+                std::env::var_os("SEARCHI_TEST_FFPROBE").context("Set SEARCHI_TEST_FFPROBE")?,
+            ),
+        )?;
+        let limits = MediaLimits::default();
+        let mut families = BTreeMap::new();
+        let mut verified = 0;
+        for candidate in plan["candidates"].as_array().context("Candidates absent")? {
+            if candidate.get("pcm_sha256").is_none() {
+                continue;
+            }
+            let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?);
+            ensure!(
+                sha(&fs::read(&path)?) == candidate["sha256"],
+                "Container changed"
+            );
+            let actual = probe(&tools, &path, &limits)?;
+            ensure!(
+                fields(&serde_json::to_value(&actual.tags)?) == fields(&candidate["actual_tags"]),
+                "Tag evidence differs"
+            );
+            let window = audio_windows(&actual, &limits)?
+                .into_iter()
+                .next()
+                .context("Audio absent")?;
+            let pcm = decode_audio(&tools, &path, window, &limits)?;
+            let bytes: Vec<_> = pcm.iter().flat_map(|value| value.to_le_bytes()).collect();
+            let hash = sha(&bytes);
+            ensure!(hash == candidate["pcm_sha256"], "Production PCM differs");
+            if let Some(previous) = families.insert(candidate["id"].to_string(), hash.clone()) {
+                ensure!(previous == hash, "Variant audio differs");
+            }
+            verified += 1;
+        }
+        ensure!(
+            verified == 18 && families.len() == 6,
+            "Expected six real three-variant families"
+        );
+        Ok(())
+    }
 }