karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Verify unseen music stress inputs against production decoding

Browse Source
This commit is contained in:
karl committed 2026-10-13 18:54:47 +02:00
1 parent d27be302c8
commit cbf07b0575
4 files changed
+90 -28
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fixtures/music-validation-v1/README.md
+12 -3
View File
@@ -55,18 +55,25 @@ metadata variants share recordings and are correlated, not independent examples.
## Run after reserving the CPU slot
```powershell
python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1 .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe
python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1-canonical .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe
cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked
$env:SEARCHI_FORCE_CPU = '1'
cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1/results.json
$env:SEARCHI_METADATA_VALIDATION_PLAN = 'C:/Users/karl/searchi/.searchi/qa/music-validation-v1-canonical/prepared.json'
$env:SEARCHI_TEST_FFMPEG = 'C:/Users/karl/searchi/.searchi/media-runtime/bin/ffmpeg.exe'
$env:SEARCHI_TEST_FFPROBE = 'C:/Users/karl/searchi/.searchi/media-runtime/bin/ffprobe.exe'
cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- --include-ignored
cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1-canonical/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1-canonical/results.json
```
Preparation already completed without inference. Its frozen plan SHA-256 is
`8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2`;
`3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78`;
fixture SHA-256 is
`f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577`.
Use a fresh output directory for another preparation; existing plans and result
files are preserved. Decoder hashes and actual container tags are checked.
Five tests passed, including actual production decoder/probe parity on all
18 stress containers; strict Clippy passed. This preflight performed no model
inference. Adjust the local fixture/helper paths for another development machine.
The diagnostic uses actual Google EmbeddingGemma 2 on CPU, one thread. Each of
the eight distinct audio sources is encoded once and both photos at 280 tokens.
@@ -76,3 +83,5 @@ citations remain explicit; a metadata hit has no fabricated timestamp. Four
lanes compare media-only, original tags, missing tags and misleading tags without
tuned weights or cutoff. Report positive recall and no-answer failure separately.
This direct cosine diagnostic is not production ANN/hybrid validation.
The earlier prepared plan 8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2 was never inferred. Pre-inference review found that its verification decoded full float PCM while the production decoder uses millisecond trimming and s16 normalization. The corrected plan above matches the actual production decoder and compares only the whitelisted tag evidence; labels, source bytes, query texts and ranking hypothesis are unchanged. The earlier plan is preserved under its original ignored directory.
fixtures/music-validation-v1/inputs.lock.json
+19 -19
View File
@@ -1,7 +1,7 @@
{
"frozen_before_inference": true,
"fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577",
"prepared_plan_sha256": "8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2",
"prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78",
"sources": [
{
"id": "unseen01",
@@ -80,7 +80,7 @@
"artist": "Ryan",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23"
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen01",
@@ -90,7 +90,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23"
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen01",
@@ -101,7 +101,7 @@
"title": "cello strings",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23"
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen02",
@@ -112,7 +112,7 @@
"title": "cello strings",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed"
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen02",
@@ -122,7 +122,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed"
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen02",
@@ -132,7 +132,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed"
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen03",
@@ -142,7 +142,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504"
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen03",
@@ -152,7 +152,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504"
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen03",
@@ -164,7 +164,7 @@
"title": "Bucket mute",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504"
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen04",
@@ -176,7 +176,7 @@
"title": "Bucket mute",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161"
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen04",
@@ -186,7 +186,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161"
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen04",
@@ -198,7 +198,7 @@
"title": "After you've gone - first 8 bars on tenor sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161"
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen05",
@@ -210,7 +210,7 @@
"title": "After you've gone - first 8 bars on tenor sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd"
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen05",
@@ -220,7 +220,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd"
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen05",
@@ -232,7 +232,7 @@
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd"
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen06",
@@ -244,7 +244,7 @@
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b"
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "unseen06",
@@ -254,7 +254,7 @@
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b"
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "unseen06",
@@ -265,7 +265,7 @@
"artist": "Ryan",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b"
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "image01",