diff --git a/fixtures/music-validation-v1/README.md b/fixtures/music-validation-v1/README.md index 1f493e2..ca064b7 100644 --- a/fixtures/music-validation-v1/README.md +++ b/fixtures/music-validation-v1/README.md @@ -55,18 +55,25 @@ metadata variants share recordings and are correlated, not independent examples. ## Run after reserving the CPU slot ```powershell -python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1 .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe +python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1-canonical .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked $env:SEARCHI_FORCE_CPU = '1' -cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1/results.json +$env:SEARCHI_METADATA_VALIDATION_PLAN = 'C:/Users/karl/searchi/.searchi/qa/music-validation-v1-canonical/prepared.json' +$env:SEARCHI_TEST_FFMPEG = 'C:/Users/karl/searchi/.searchi/media-runtime/bin/ffmpeg.exe' +$env:SEARCHI_TEST_FFPROBE = 'C:/Users/karl/searchi/.searchi/media-runtime/bin/ffprobe.exe' +cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- --include-ignored +cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1-canonical/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1-canonical/results.json ``` Preparation already completed without inference. Its frozen plan SHA-256 is -`8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2`; +`3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78`; fixture SHA-256 is `f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577`. Use a fresh output directory for another preparation; existing plans and result files are preserved. Decoder hashes and actual container tags are checked. +Five tests passed, including actual production decoder/probe parity on all +18 stress containers; strict Clippy passed. This preflight performed no model +inference. Adjust the local fixture/helper paths for another development machine. The diagnostic uses actual Google EmbeddingGemma 2 on CPU, one thread. Each of the eight distinct audio sources is encoded once and both photos at 280 tokens. @@ -76,3 +83,5 @@ citations remain explicit; a metadata hit has no fabricated timestamp. Four lanes compare media-only, original tags, missing tags and misleading tags without tuned weights or cutoff. Report positive recall and no-answer failure separately. This direct cosine diagnostic is not production ANN/hybrid validation. + +The earlier prepared plan 8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2 was never inferred. Pre-inference review found that its verification decoded full float PCM while the production decoder uses millisecond trimming and s16 normalization. The corrected plan above matches the actual production decoder and compares only the whitelisted tag evidence; labels, source bytes, query texts and ranking hypothesis are unchanged. The earlier plan is preserved under its original ignored directory. diff --git a/fixtures/music-validation-v1/inputs.lock.json b/fixtures/music-validation-v1/inputs.lock.json index e6ceaed..2c09aea 100644 --- a/fixtures/music-validation-v1/inputs.lock.json +++ b/fixtures/music-validation-v1/inputs.lock.json @@ -1,7 +1,7 @@ { "frozen_before_inference": true, "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", - "prepared_plan_sha256": "8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2", + "prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78", "sources": [ { "id": "unseen01", @@ -80,7 +80,7 @@ "artist": "Ryan", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23" + "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" }, { "id": "unseen01", @@ -90,7 +90,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23" + "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" }, { "id": "unseen01", @@ -101,7 +101,7 @@ "title": "cello strings", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23" + "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" }, { "id": "unseen02", @@ -112,7 +112,7 @@ "title": "cello strings", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed" + "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" }, { "id": "unseen02", @@ -122,7 +122,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed" + "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" }, { "id": "unseen02", @@ -132,7 +132,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed" + "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" }, { "id": "unseen03", @@ -142,7 +142,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504" + "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" }, { "id": "unseen03", @@ -152,7 +152,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504" + "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" }, { "id": "unseen03", @@ -164,7 +164,7 @@ "title": "Bucket mute", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504" + "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" }, { "id": "unseen04", @@ -176,7 +176,7 @@ "title": "Bucket mute", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161" + "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" }, { "id": "unseen04", @@ -186,7 +186,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161" + "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" }, { "id": "unseen04", @@ -198,7 +198,7 @@ "title": "After you've gone - first 8 bars on tenor sax", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161" + "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" }, { "id": "unseen05", @@ -210,7 +210,7 @@ "title": "After you've gone - first 8 bars on tenor sax", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd" + "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" }, { "id": "unseen05", @@ -220,7 +220,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd" + "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" }, { "id": "unseen05", @@ -232,7 +232,7 @@ "title": "Ain't She Sweet, first 8 bars on soprano sax", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd" + "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" }, { "id": "unseen06", @@ -244,7 +244,7 @@ "title": "Ain't She Sweet, first 8 bars on soprano sax", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b" + "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" }, { "id": "unseen06", @@ -254,7 +254,7 @@ "actual_tags": { "encoder": "Lavf63.1.102" }, - "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b" + "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" }, { "id": "unseen06", @@ -265,7 +265,7 @@ "artist": "Ryan", "encoder": "Lavf63.1.102" }, - "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b" + "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" }, { "id": "image01", diff --git a/tools/media-eval/prepare_music_validation.py b/tools/media-eval/prepare_music_validation.py index a9031bb..a13ddc8 100644 --- a/tools/media-eval/prepare_music_validation.py +++ b/tools/media-eval/prepare_music_validation.py @@ -4,6 +4,7 @@ import hashlib import json from pathlib import Path import subprocess +import struct import urllib.request @@ -23,11 +24,13 @@ def probe(path, ffprobe): return tags -def pcm(path, ffmpeg): - data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-t", "30", "-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"]) - if not data or len(data) > 480000 * 4 or len(data) % 4: +def pcm(path, ffmpeg, ffprobe): + metadata = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-of", "json", str(path)])) + duration_ms = min(30000, int(float(metadata["format"]["duration"]) * 1000 + 0.5)) + data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-ss", "0.000", "-t", f"{duration_ms / 1000:.3f}", "-i", str(path), "-af", f"aresample=16000,atrim=end_sample={duration_ms * 16},asetpts=PTS-STARTPTS", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le", "pipe:1"]) + if not data or len(data) > 480000 * 2 or len(data) % 2: raise ValueError("Invalid bounded PCM") - return sha(data) + return sha(b"".join(struct.pack(" Result<()> { if candidate["kind"] == "audio" { let actual = probe(&tools, &path, &limits)?; ensure!( - serde_json::to_value(&actual.tags)? == candidate["actual_tags"], + fields(&serde_json::to_value(&actual.tags)?) == fields(&candidate["actual_tags"]), "Actual tags differ" ); let window = audio_windows(&actual, &limits)? @@ -215,4 +215,54 @@ mod tests { assert!(dot(&[0.0], &[0.0]).is_err()); assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err()); } + #[test] + #[ignore = "Requires the real prepared containers and CPU FFmpeg; no model inference"] + fn prepared_pcm_and_tags_match_production_decoder() -> Result<()> { + let plan_path = std::env::var_os("SEARCHI_METADATA_VALIDATION_PLAN") + .context("Set SEARCHI_METADATA_VALIDATION_PLAN")?; + let plan: Value = serde_json::from_slice(&fs::read(plan_path)?)?; + let tools = FfmpegTools::new( + PathBuf::from( + std::env::var_os("SEARCHI_TEST_FFMPEG").context("Set SEARCHI_TEST_FFMPEG")?, + ), + PathBuf::from( + std::env::var_os("SEARCHI_TEST_FFPROBE").context("Set SEARCHI_TEST_FFPROBE")?, + ), + )?; + let limits = MediaLimits::default(); + let mut families = BTreeMap::new(); + let mut verified = 0; + for candidate in plan["candidates"].as_array().context("Candidates absent")? { + if candidate.get("pcm_sha256").is_none() { + continue; + } + let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?); + ensure!( + sha(&fs::read(&path)?) == candidate["sha256"], + "Container changed" + ); + let actual = probe(&tools, &path, &limits)?; + ensure!( + fields(&serde_json::to_value(&actual.tags)?) == fields(&candidate["actual_tags"]), + "Tag evidence differs" + ); + let window = audio_windows(&actual, &limits)? + .into_iter() + .next() + .context("Audio absent")?; + let pcm = decode_audio(&tools, &path, window, &limits)?; + let bytes: Vec<_> = pcm.iter().flat_map(|value| value.to_le_bytes()).collect(); + let hash = sha(&bytes); + ensure!(hash == candidate["pcm_sha256"], "Production PCM differs"); + if let Some(previous) = families.insert(candidate["id"].to_string(), hash.clone()) { + ensure!(previous == hash, "Variant audio differs"); + } + verified += 1; + } + ensure!( + verified == 18 && families.len() == 6, + "Expected six real three-variant families" + ); + Ok(()) + } }