From d27be302c882cb5f82fc5210a66c5b42c94c21f4 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 18:51:50 +0200 Subject: [PATCH] Freeze unseen music stress corpus and independently verified gated DTW experiment --- docs/SPEECH-DTW.md | 88 +++++ fixtures/music-validation-v1/README.md | 78 +++++ fixtures/music-validation-v1/fixture.json | 268 ++++++++++++++++ fixtures/music-validation-v1/inputs.lock.json | 303 ++++++++++++++++++ fixtures/music-validation-v1/labels.lock.json | 6 + tools/media-eval/prepare_music_validation.py | 104 ++++++ .../media-eval/src/bin/metadata_validation.rs | 218 +++++++++++++ tools/speech-rank/README.md | 10 + tools/speech-rank/src/alignment.rs | 44 ++- tools/speech-rank/src/main.rs | 66 ++++ tools/speech-vad/README.md | 33 ++ tools/speech-vad/gated-dtw-policy.json | 13 + tools/speech-vad/gated_asr.py | 71 +++- tools/speech-vad/test_gated_asr.py | 9 + 14 files changed, 1305 insertions(+), 6 deletions(-) create mode 100644 docs/SPEECH-DTW.md create mode 100644 fixtures/music-validation-v1/README.md create mode 100644 fixtures/music-validation-v1/fixture.json create mode 100644 fixtures/music-validation-v1/inputs.lock.json create mode 100644 fixtures/music-validation-v1/labels.lock.json create mode 100644 tools/media-eval/prepare_music_validation.py create mode 100644 tools/media-eval/src/bin/metadata_validation.rs create mode 100644 tools/speech-vad/gated-dtw-policy.json diff --git a/docs/SPEECH-DTW.md b/docs/SPEECH-DTW.md new file mode 100644 index 0000000..6c33242 --- /dev/null +++ b/docs/SPEECH-DTW.md @@ -0,0 +1,88 @@ +# DTW diagnostic: strict UTF-8 failure + +Status: internal CPU diagnosis, 2026-10-13. No accepted speech/retrieval score, +production ASR integration or new media rating follows from this evidence. +Google EmbeddingGemma 2 remains the production semantic model. + +The original fifteen-window DTW experiment stopped on strict UTF-8 parsing. +Its temporary raw output was cleaned, so that first run's exact bytes could not +be inspected. Its partial report remains unchanged. A separate coordinator- +authorized diagnostic reproduced exactly the second `music01` window: source +offset 30000 ms, duration 30000 ms, primary audio stream, 480000 decoded samples. +No labels selected text/timestamps, and the whole experiment was not rerun. + +## Revalidated execution + +The whisper.cpp checkout was clean at pinned commit +`d1be6fde11ac6e0407606b4e42fe72d34add8037`. Before execution, the fixture and +source hashes, base model size/hash, worker hash and both decoder hashes were +checked against frozen evidence. The owned runner retained raw output, WAV, +probe, stderr and report directly under ignored +`.searchi/speech/dtw-window-diagnostic-v1`; no temporary cleanup removed them. + +| Artifact | SHA-256 | +| --- | --- | +| Frozen fixture | `b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393` | +| Source music01 | `29fcd20414ee19c2a5aa4339eaf6375079dc87959051ab0aea32c76df52f34dc` | +| CPU worker | `12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8` | +| Multilingual base model | `60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe` | +| FFmpeg | `b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46` | +| FFprobe | `4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236` | +| Raw JSON, 29148 bytes | `f1de2c295b0c9a223619abe6dbf34823be6a6aa3dbcad69fea1d6121fa0c200e` | +| Native log | `28d174198a3ef980aa5c8c1237618f450a19b0a80c8a270209dce113a8de3a7a` | +| Diagnostic report | `b4b0dedfac6365c59793806801b1fc5d9f8f203891b623fb86ad508352ad0114` | + +Windows VM execution forced CPU: `SEARCHI_FORCE_CPU=1`, native flags +`--no-gpu --threads 1 --processors 1 --language auto --output-json-full +--dtw base --no-flash-attn`, verified base model and the decoded owned WAV. +The existing bounded decoder used absolute 30-second seeking and sample trimming; +its output contained exactly 480000 samples. Native ASR exited zero after +42493.25 ms; decoding took 271.42 ms. The owned diagnostic runner exited zero +after recording `strict_parse: rejected`. This is not a successful ASR result +or a warmed throughput measurement. No other model inference ran concurrently +under this coordinator's CPU slot; unrelated VM load was not instrumented. + +## Exact failure and pinned native source + +Strict UTF-8 decoding rejects bytes **1137–1138** (zero-based half-open range +1137–1139). The segment text ends with bytes `E2 80` immediately before the +closing JSON quote: a three-byte UTF-8 sequence has no final continuation byte. +The last ordinary token, id 913, also ends with exactly `E2 80` at byte 28900. +Earlier token text contains complete `E2 80 8D` codepoints. Raw bytes were +inspected directly, without replacement decoding or writing repaired JSON. + +The pinned native core constructs segment text by concatenating token strings. +The CLI full-JSON serializer merges adjacent token byte pieces while a codepoint +needs continuation **and another token exists**. At the last token there is no +continuation to merge; it writes the remaining incomplete bytes. Its segment +text serializer likewise writes the already incomplete segment bytes. The +escaping function only escapes quotes/backslashes; it cannot restore a missing +codepoint byte. Thus this is incomplete recognized byte output, reproduced in +both segment and token text, rather than a Python ANSI conversion or DTW time +unit error. The exact internal reason the decoder terminated this token sequence +has not been instrumented and remains unproven. + +Relevant pinned source: [CLI serializer and token merging](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/examples/cli/cli.cpp), +[UTF-8 continuation helper](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/examples/common-whisper.cpp), +and [segment construction](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/src/whisper.cpp). + +## Minimal next change and limits + +Before changing a native worker, add a strict UTF-8 validator at its JSON boundary +for original segment strings and merged token strings. If original recognized +bytes remain invalid after adjacent-token merging, return an explicit typed +recognition-output error with bounded raw evidence; never replace characters, +truncate text, synthesize missing bytes or fabricate timestamps. Test a valid +split codepoint and an incomplete final codepoint. This makes the failure +machine-readable; it does **not** repair recognition or establish improved +retrieval. An alternative timestamp/recognition policy must be declared and +evaluated separately before any default changes. + +The Python harness already fails closed, retains bounded raw bytes with SHA and +parse positions, and atomically records an incomplete, ineligible checkpoint. +The original partial DTW report contains three NASA windows with 21 valid +point-based sentence spans and one separately retained ordinary-interval error. +It lacks remaining piano windows and three other competitors. Neither that +partial report nor this one-window diagnosis can receive an accepted quality +score. Packaging, independent bilingual speech, silence/music hallucination, +resource peaks and actual production hybrid integration remain open. diff --git a/fixtures/music-validation-v1/README.md b/fixtures/music-validation-v1/README.md new file mode 100644 index 0000000..1f493e2 --- /dev/null +++ b/fixtures/music-validation-v1/README.md @@ -0,0 +1,78 @@ +# Unseen music metadata stress validation v1 + +This small source pool and its 18 queries were frozen before any model inference. +It is independent of the two music files used to motivate the metadata-field +hypothesis. Labels come from publisher descriptions, not model outputs. There is +one assistant annotator; human listening and a blinded second annotator have not +been executed. These instrument demos and short excerpts do not represent a +large music collection or full-song recognition. + +| Source | Publisher / attribution | License | Download bytes | +| --- | --- | --- | ---: | +| [Acoustic guitar sample](https://commons.wikimedia.org/wiki/File:AcousticGuitarSample.ogg) | RyGuy / Ryan | PD-self | 889069 | +| [Cello strings](https://commons.wikimedia.org/wiki/File:Cello_strings.ogg) | Georg Feitscher | CC BY-SA 3.0 | 296474 | +| [Drum cadence A](https://commons.wikimedia.org/wiki/File:Drum_-_Cadence_A.ogg) | United States Navy Band | PD-USGov | 1574547 | +| [Bucket mute trumpet demo](https://commons.wikimedia.org/wiki/File:Bucket_mute_trumpet_demo.ogg) | Ovinus Real | CC BY-SA 4.0 | 228972 | +| [After You've Gone, tenor saxophone excerpt](https://commons.wikimedia.org/wiki/File:Afteryouvegone-first8bars.ogg) | Julien Grandgagnage / Beachcomber | CC BY-SA 3.0 | 300001 | +| [Ain't She Sweet, soprano saxophone excerpt](https://commons.wikimedia.org/wiki/File:Aintshesweet8bars.ogg) | Julien Grandgagnage / Beachcomber | CC BY-SA 3.0 | 250846 | + +Total acquisition: **3539909 bytes**, approximately 3.38 MiB. License URLs and +publisher attribution are preserved in `fixture.json`; source SHA-1 comes from +the official Commons imageinfo API, acquired bytes are also SHA-256 locked in +`inputs.lock.json`. Do not substitute similarly named files. Raw audio and +generated containers stay under ignored `.searchi/qa/music-validation-v1`. + +## Preparation and stress variants + +Preparation takes the first at most 30 seconds, resamples to mono 16 kHz and +encodes lossless FLAC. It produces actual containers with original whitelisted +tags, missing tags and cyclically transferred actual tags from the next source. +The exact decoded PCM SHA-256 must match across each source's three variants +before audio vectors can be reused. These are controlled tag-corruption cases, +not naturally mislabeled customer files. Generated excerpts/variants retain the +original source's license and attribution, with these modifications disclosed. + +The drum source actually has no descriptive tags. The guitar has only artist +`Ryan`. Cello supplies `cello strings`; trumpet supplies `Bucket mute`; the sax +titles identify tenor/soprano. Empty tag input creates no metadata vector. A +cyclic transfer from the tagless drum remains tagless; this fact is retained, +not replaced with an invented instrument label. Comments and encoder strings +are not semantic metadata input. + +Two existing photo and two sound-effect files are fixed mixed-media competitors: +`image01`, `image02`, `sound01`, `sound02` from the previously frozen media corpus. +They are distractors, not new validation positives. All competitors stay in each +lane. This does not cover the entire 32-file mixed regression or spoken videos. + +Twelve instrument queries have English/German pairs. Broad saxophone queries +credit both sax recordings; they are not incorrectly declared absent. Four +absence queries ask for bagpipes, an absent singer/title, and an absent spoken +topic. They remain separate from positive recall; no threshold has been selected. +Specific tenor/soprano labels follow primary descriptions, with listening still +needed to independently verify perceptual specificity. Language pairs and +metadata variants share recordings and are correlated, not independent examples. + +## Run after reserving the CPU slot + +```powershell +python tools/media-eval/prepare_music_validation.py fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1 .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe +cargo test --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked +$env:SEARCHI_FORCE_CPU = '1' +cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata_validation --locked -- fixtures/music-validation-v1/fixture.json .searchi/qa/music-validation-v1/prepared.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-validation-v1/results.json +``` + +Preparation already completed without inference. Its frozen plan SHA-256 is +`8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2`; +fixture SHA-256 is +`f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577`. +Use a fresh output directory for another preparation; existing plans and result +files are preserved. Decoder hashes and actual container tags are checked. + +The diagnostic uses actual Google EmbeddingGemma 2 on CPU, one thread. Each of +the eight distinct audio sources is encoded once and both photos at 280 tokens. +Metadata fields use neutral `File metadata`, the unchanged 2048-byte whitelist +policy and maximum over each file's real evidence. All field, source and variant +citations remain explicit; a metadata hit has no fabricated timestamp. Four +lanes compare media-only, original tags, missing tags and misleading tags without +tuned weights or cutoff. Report positive recall and no-answer failure separately. +This direct cosine diagnostic is not production ANN/hybrid validation. diff --git a/fixtures/music-validation-v1/fixture.json b/fixtures/music-validation-v1/fixture.json new file mode 100644 index 0000000..6750ce8 --- /dev/null +++ b/fixtures/music-validation-v1/fixture.json @@ -0,0 +1,268 @@ +{ + "version": 1, + "frozen_before_inference": true, + "assets": [ + { + "id": "unseen01", + "filename": "AcousticGuitarSample.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:AcousticGuitarSample.ogg", + "bytes": 889069, + "sha1": "d622d15b17b5c101631dd7707a57cf24b9c474da", + "license": "Public domain", + "license_url": null, + "credit_html": "RyGuy", + "publisher_description_html": "Sample of the sounds which can be made by an acoustic guitar.", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen02", + "filename": "Cello strings.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Cello_strings.ogg", + "bytes": 296474, + "sha1": "8c965bc8679ac6b84b6abeaa086311892d590cae", + "license": "CC BY-SA 3.0", + "license_url": "http://creativecommons.org/licenses/by-sa/3.0/", + "credit_html": null, + "publisher_description_html": null, + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen03", + "filename": "Drum - Cadence A.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Drum_-_Cadence_A.ogg", + "bytes": 1574547, + "sha1": "19d4fbf201e4990d9b2be9c454b209f36af86700", + "license": "Public domain", + "license_url": null, + "credit_html": "Unknown authorUnknown author", + "publisher_description_html": "Drum Cadence A performed by the United States Navy Band", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen04", + "filename": "Bucket mute trumpet demo.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Bucket_mute_trumpet_demo.ogg", + "bytes": 228972, + "sha1": "37ebe58d5d3eb6051b066ed31bcc66240518b86c", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "credit_html": "Ovinus Real", + "publisher_description_html": "I play the concert notes F3, Bb3, Eb4, Ab4, Db5, Gb5, B5 in succession on a Bb trumpet with bucket mute inserted. I then play a short jazz-ish excerpt (that I had previously improvised) to demonstrate the character of the tone. The sound was amplified but not processed in any other way.", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen05", + "filename": "Afteryouvegone-first8bars.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Afteryouvegone-first8bars.ogg", + "bytes": 300001, + "sha1": "92eee317927ff5cdc5614fe848350cfde2419df3", + "license": "CC BY-SA 3.0", + "license_url": "https://creativecommons.org/licenses/by-sa/3.0", + "credit_html": "Julien Grandgagnage", + "publisher_description_html": "After You've Gone - first 8 bars on tenor saxophone", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen06", + "filename": "Aintshesweet8bars.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Aintshesweet8bars.ogg", + "bytes": 250846, + "sha1": "3fb787158e3250e8a30f99d82fd9ee0f4743511c", + "license": "CC BY-SA 3.0", + "license_url": "https://creativecommons.org/licenses/by-sa/3.0", + "credit_html": "Julien Grandgagnage", + "publisher_description_html": "Ain't She Sweet, first 8 bars on soprano saxophone", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + } + ], + "queries": [ + { + "id": "unseen01-en", + "text": "a recording of acoustic guitar", + "language": "en", + "expected": [ + "unseen01" + ], + "category": "instrument" + }, + { + "id": "unseen01-de", + "text": "eine Aufnahme mit akustische Gitarre", + "language": "de", + "expected": [ + "unseen01" + ], + "category": "instrument" + }, + { + "id": "unseen02-en", + "text": "a recording of cello strings", + "language": "en", + "expected": [ + "unseen02" + ], + "category": "instrument" + }, + { + "id": "unseen02-de", + "text": "eine Aufnahme mit Cellosaiten", + "language": "de", + "expected": [ + "unseen02" + ], + "category": "instrument" + }, + { + "id": "unseen03-en", + "text": "a recording of military drum cadence", + "language": "en", + "expected": [ + "unseen03" + ], + "category": "instrument" + }, + { + "id": "unseen03-de", + "text": "eine Aufnahme mit militärische Trommelkadenz", + "language": "de", + "expected": [ + "unseen03" + ], + "category": "instrument" + }, + { + "id": "unseen04-en", + "text": "a recording of trumpet with a bucket mute", + "language": "en", + "expected": [ + "unseen04" + ], + "category": "instrument" + }, + { + "id": "unseen04-de", + "text": "eine Aufnahme mit Trompete mit Bucket-Dämpfer", + "language": "de", + "expected": [ + "unseen04" + ], + "category": "instrument" + }, + { + "id": "unseen05-en", + "text": "a recording of tenor saxophone melody", + "language": "en", + "expected": [ + "unseen05" + ], + "category": "instrument" + }, + { + "id": "unseen05-de", + "text": "eine Aufnahme mit Tenorsaxophon-Melodie", + "language": "de", + "expected": [ + "unseen05" + ], + "category": "instrument" + }, + { + "id": "unseen06-en", + "text": "a recording of soprano saxophone melody", + "language": "en", + "expected": [ + "unseen06" + ], + "category": "instrument" + }, + { + "id": "unseen06-de", + "text": "eine Aufnahme mit Sopransaxophon-Melodie", + "language": "de", + "expected": [ + "unseen06" + ], + "category": "instrument" + }, + { + "id": "sax-broad-en", + "text": "a saxophone playing a melody", + "language": "en", + "expected": [ + "unseen05", + "unseen06" + ], + "category": "related-positive" + }, + { + "id": "sax-broad-de", + "text": "eine Saxophonmelodie", + "language": "de", + "expected": [ + "unseen05", + "unseen06" + ], + "category": "related-positive" + }, + { + "id": "absent-bagpipe-en", + "text": "a bagpipe performance", + "language": "en", + "expected": [], + "category": "absent-instrument" + }, + { + "id": "absent-bagpipe-de", + "text": "eine Dudelsackaufnahme", + "language": "de", + "expected": [], + "category": "absent-instrument" + }, + { + "id": "absent-artist-en", + "text": "Adele singing Hello", + "language": "en", + "expected": [], + "category": "absent-identity" + }, + { + "id": "absent-content-en", + "text": "a person explaining quantum entanglement", + "language": "en", + "expected": [], + "category": "absent-content" + } + ], + "variants": [ + "original", + "missing_metadata", + "misleading_metadata" + ], + "misleading_policy": "Whitelisted actual tags cyclically transferred from the next asset; no invented descriptive tags. All clips encode first <=30seconds as lossless FLAC mono16kHz, metadata stripped or replaced. Verify exact decoded PCM SHA256 equality across variants before reusing audio vectors. Report controlled corruption separately.", + "candidate_policy": "New six sources plus optional predeclared legacy image/audio/video distractors, fixed before inference; never treat related plausible music as absent. Broad sax labels credit both tenor and soprano. Source title is not instrument evidence for the raw-audio lane.", + "metadata_policy": "Whitelisted artist/title/album/genre/composer fields separately, total2048 UTF8bytes, neutral File metadata title, file maximum over real metadata and audio, noweights/nocutoff.", + "legacy_distractors": [ + "image01", + "image02", + "sound01", + "sound02" + ] +} diff --git a/fixtures/music-validation-v1/inputs.lock.json b/fixtures/music-validation-v1/inputs.lock.json new file mode 100644 index 0000000..e6ceaed --- /dev/null +++ b/fixtures/music-validation-v1/inputs.lock.json @@ -0,0 +1,303 @@ +{ + "frozen_before_inference": true, + "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", + "prepared_plan_sha256": "8598312836f3c027b578f9f93ca1cd2d38d708bc0fd713fe174f10776934cbb2", + "sources": [ + { + "id": "unseen01", + "sha256": "1c604cce5299b2f9579538810deb328d9c47607f098f5885ff1daa7cf34b637a", + "bytes": 889069, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", + "actual_tags": { + "artist": "Ryan" + } + }, + { + "id": "unseen02", + "sha256": "f62b0dabedd570f3abd9fc9e9a3a468375bb77ddd7f1ad02c203c8d5394bf219", + "bytes": 296474, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg?download=1", + "actual_tags": { + "title": "cello strings", + "date": "06/10/2005" + } + }, + { + "id": "unseen03", + "sha256": "5083f07730c6ffb1cd2c3b5cc1a43714a0cfda04fe913935e83d1048466562e6", + "bytes": 1574547, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg?download=1", + "actual_tags": {} + }, + { + "id": "unseen04", + "sha256": "7fc61102e3050c5fe01000b1b02f839e725b305333505075916faeea78844caa", + "bytes": 228972, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg?download=1", + "actual_tags": { + "artist": "Ovinus Real", + "comments": "First played a series of ascending fourths from concert F3 to B5, then replayed the improvised excerpt from the harmon mute (without stem) recording. No processing or noise removal done.", + "date": "2020", + "title": "Bucket mute" + } + }, + { + "id": "unseen05", + "sha256": "0570623af1490ed81b86c5a95f5b95eb61b9d941bb0116e4271e61a07d2dc5ba", + "bytes": 300001, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg?download=1", + "actual_tags": { + "title": "After you've gone - first 8 bars on tenor sax", + "date": "2013", + "artist": "Beachcomber" + } + }, + { + "id": "unseen06", + "sha256": "8f2035ab8924ee9f0d212de5853cf22eab7457520c1d774413a78601ed1b8bd6", + "bytes": 250846, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg?download=1", + "actual_tags": { + "title": "Ain't She Sweet, first 8 bars on soprano sax", + "date": "2013", + "artist": "Beachcomber" + } + } + ], + "containers": [ + { + "id": "unseen01", + "variant": "original", + "kind": "audio", + "sha256": "24b14ccc428e96312a96a556e5250b3d4c781bb42e4549f13b43ddc2fb31c087", + "actual_tags": { + "artist": "Ryan", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23" + }, + { + "id": "unseen01", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "665ddf557d94a65ff98f1d35563657405846577351de08f00b5a104bb1d6f992", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23" + }, + { + "id": "unseen01", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "bac63053fc9182e17269ec1015ebf2bc74b803abfb4f52c25a0009f145c7065a", + "actual_tags": { + "title": "cello strings", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "0df617517cc3900ec0bf8612985d8821ae633a50ac886fcac50c9fe79dd27d23" + }, + { + "id": "unseen02", + "variant": "original", + "kind": "audio", + "sha256": "fa934449cb45743ffa420c412e3b7bc388ae846e76b29573922ab9506496bbfb", + "actual_tags": { + "title": "cello strings", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed" + }, + { + "id": "unseen02", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed" + }, + { + "id": "unseen02", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "7360f456a453e113f05f78ee7a496bd2f6ed412c6a3bef28c0a7c93f91c3efed" + }, + { + "id": "unseen03", + "variant": "original", + "kind": "audio", + "sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504" + }, + { + "id": "unseen03", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504" + }, + { + "id": "unseen03", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "5e108abbbc44fe068b07a5b124894c4738204f511a308c4385960019f5c453a1", + "actual_tags": { + "artist": "Ovinus Real", + "title": "Bucket mute", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "ff2e6123d2b6d700e00a77d603181c485477d005c46c858db10f61339a33d504" + }, + { + "id": "unseen04", + "variant": "original", + "kind": "audio", + "sha256": "fc498bc0e47fc97b8c085cd00266d3e2dbebecfc3021e443b7b19e43dd025dde", + "actual_tags": { + "artist": "Ovinus Real", + "title": "Bucket mute", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161" + }, + { + "id": "unseen04", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "09b2df6db7953ed67e5ae5059b3adacbec7e5c13a3be6aa32237b19e106e9fac", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161" + }, + { + "id": "unseen04", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "fdaae3a8636394100b268a4a2ea8b42565be87fcb8fbdaefc9bee29a42d77b8a", + "actual_tags": { + "artist": "Beachcomber", + "title": "After you've gone - first 8 bars on tenor sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "3b342b2cf7ddc0c19fed1690c8a5a513dc66e1751b522bfd6fc293a52222b161" + }, + { + "id": "unseen05", + "variant": "original", + "kind": "audio", + "sha256": "caa19f57058fb7a1d7ac715c087a6c8430f9054b7a3c503ab7b191794f805652", + "actual_tags": { + "artist": "Beachcomber", + "title": "After you've gone - first 8 bars on tenor sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd" + }, + { + "id": "unseen05", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "1e46c0ec9c8831fa041d27c46cba924803a29fe5304bf5776d598db297b0b24f", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd" + }, + { + "id": "unseen05", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "56217ac0a4a85921e939c7a84fbe864e1d8c9e1d7b48b6ba54d64187370423aa", + "actual_tags": { + "artist": "Beachcomber", + "title": "Ain't She Sweet, first 8 bars on soprano sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "9882f6bb8f59886f26bfc09c9059adec357018c40b9c04e7ccb4dc7097cb2dbd" + }, + { + "id": "unseen06", + "variant": "original", + "kind": "audio", + "sha256": "27b4a28c800ac7b949af7f92d1e483bc4089c98a45a381b52c816254ef3810fc", + "actual_tags": { + "artist": "Beachcomber", + "title": "Ain't She Sweet, first 8 bars on soprano sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b" + }, + { + "id": "unseen06", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "603b253da7ee235284ab8bb9cdac177001093122ce385032b6a2a00d4ffcb1e8", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b" + }, + { + "id": "unseen06", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "f8b0e7f4261f69fa326370433dfd6519c09e9a63f12fcaf9dbf47fe8cf481fe2", + "actual_tags": { + "artist": "Ryan", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "7f04f029e66ebf59e7cc39d04d6e30895f5b727ca055206212522732bc42834b" + }, + { + "id": "image01", + "variant": "fixed", + "kind": "image", + "sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed", + "actual_tags": {} + }, + { + "id": "image02", + "variant": "fixed", + "kind": "image", + "sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1", + "actual_tags": {} + }, + { + "id": "sound01", + "variant": "fixed", + "kind": "audio", + "sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451", + "actual_tags": {} + }, + { + "id": "sound02", + "variant": "fixed", + "kind": "audio", + "sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e", + "actual_tags": { + "encoder": "LAME3.98r" + } + } + ], + "ffmpeg_sha256": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46", + "ffprobe_sha256": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236" +} diff --git a/fixtures/music-validation-v1/labels.lock.json b/fixtures/music-validation-v1/labels.lock.json new file mode 100644 index 0000000..b224296 --- /dev/null +++ b/fixtures/music-validation-v1/labels.lock.json @@ -0,0 +1,6 @@ +{ + "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", + "frozen_before_inference": true, + "query_count": 18, + "asset_count": 6 +} diff --git a/tools/media-eval/prepare_music_validation.py b/tools/media-eval/prepare_music_validation.py new file mode 100644 index 0000000..a9031bb --- /dev/null +++ b/tools/media-eval/prepare_music_validation.py @@ -0,0 +1,104 @@ +"""Bounded source acquisition and lossless metadata stress variants; no inference.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import urllib.request + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def execute(args): + return subprocess.run(args, check=True, capture_output=True, timeout=60).stdout + + +def probe(path, ffprobe): + value = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)])) + tags = {k.lower(): v for k, v in value.get("format", {}).get("tags", {}).items()} + for stream in value["streams"]: + tags.update({k.lower(): v for k, v in stream.get("tags", {}).items()}) + return tags + + +def pcm(path, ffmpeg): + data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-t", "30", "-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"]) + if not data or len(data) > 480000 * 4 or len(data) % 4: + raise ValueError("Invalid bounded PCM") + return sha(data) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("fixture", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("ffmpeg", type=Path) + parser.add_argument("ffprobe", type=Path) + args = parser.parse_args() + raw = args.fixture.read_bytes() + fixture = json.loads(raw) + lock = json.loads(args.fixture.with_name("labels.lock.json").read_text()) + if sha(raw) != lock["fixture_sha256"]: + raise ValueError("Frozen fixture changed") + args.output.mkdir(parents=True, exist_ok=True) + plan_path = args.output / "prepared.json" + if plan_path.exists(): + raise ValueError("Preserve existing prepared plan; use a fresh output directory") + sources = [] + for asset in fixture["assets"]: + destination = args.output / (asset["id"] + "-source.ogg") + if destination.exists(): + data = destination.read_bytes() + else: + request = urllib.request.Request(asset["url"] + "?download=1", headers={"User-Agent": "SearchiMediaEvaluation/1.0"}) + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read(asset["bytes"] + 1) + if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]: + raise ValueError("Publisher bytes changed") + destination.write_bytes(data) + if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]: + raise ValueError("Existing source changed") + sources.append({"asset": asset, "path": destination.resolve(), "sha256": sha(data), "tags": probe(destination, args.ffprobe)}) + prepared = [] + whitelist = ["artist", "title", "album", "genre", "composer"] + for i, source in enumerate(sources): + original_pcm = None + for variant in fixture["variants"]: + path = args.output / (source["asset"]["id"] + "-" + variant + ".flac") + tags = source["tags"] if variant == "original" else (sources[(i + 1) % len(sources)]["tags"] if variant == "misleading_metadata" else {}) + command = [str(args.ffmpeg), "-v", "error", "-nostdin", "-n", "-threads", "1", "-i", str(source["path"]), "-t", "30", "-ac", "1", "-ar", "16000", "-map_metadata", "-1", "-c:a", "flac"] + for key in whitelist: + if key in tags: + command.extend(["-metadata", f"{key}={tags[key]}"]) + command.append(str(path)) + execute(command) + pcm_hash = pcm(path, args.ffmpeg) + if original_pcm is None: + original_pcm = pcm_hash + if pcm_hash != original_pcm: + raise ValueError("Metadata variant changed decoded audio") + prepared.append({"id": source["asset"]["id"], "variant": variant, "kind": "audio", "path": str(path.resolve()), "sha256": sha(path.read_bytes()), "pcm_sha256": pcm_hash, "actual_tags": probe(path, args.ffprobe), "source_sha256": source["sha256"], "source_page": source["asset"]["source_page"], "license": source["asset"]["license"]}) + # Previously evaluated assets are distractors only, never new positive evidence. + legacy_root = Path("fixtures/media") + legacy_raw = (legacy_root / "fixture.json").read_bytes() + legacy = json.loads(legacy_raw) + legacy_lock = json.loads((legacy_root / "labels.lock.json").read_text()) + if sha(legacy_raw) != legacy_lock["fixture_sha256"]: + raise ValueError("Legacy distractor fixture changed") + for asset in legacy["assets"]: + if asset["id"] not in fixture["legacy_distractors"]: + continue + path = (legacy_root / asset["path"]).resolve() + if sha(path.read_bytes()) != asset["sha256"]: + raise ValueError("Legacy distractor changed") + prepared.append({"id": asset["id"], "variant": "fixed", "kind": "image" if asset["kind"] == "image" else "audio", "path": str(path), "sha256": asset["sha256"], "actual_tags": {} if asset["kind"] == "image" else probe(path, args.ffprobe), "legacy_distractor": True}) + result = {"fixture_sha256": sha(raw), "fixture": fixture, "candidates": prepared, "ffmpeg_sha256": sha(args.ffmpeg.read_bytes()), "ffprobe_sha256": sha(args.ffprobe.read_bytes()), "source_info": [{**s, "path": str(s["path"])} for s in sources]} + with plan_path.open("x", encoding="utf-8") as output: + json.dump(result, output, ensure_ascii=False, indent=2) + print("Prepared", len(prepared), "containers; no inference. Plan SHA256", sha(plan_path.read_bytes())) + + +if __name__ == "__main__": + main() diff --git a/tools/media-eval/src/bin/metadata_validation.rs b/tools/media-eval/src/bin/metadata_validation.rs new file mode 100644 index 0000000..723c1b7 --- /dev/null +++ b/tools/media-eval/src/bin/metadata_validation.rs @@ -0,0 +1,218 @@ +//! Mixed-pool actual metadata stress diagnostic; frozen labels never enter encodes. +use anyhow::{ensure, Context, Result}; +use searchi_media::{audio_windows, decode_audio, probe, FfmpegTools, MediaLimits}; +use searchi_semantic::{Device, GemmaEmbedder}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, io::Write, path::PathBuf, time::Instant}; +fn sha(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn fields(tags: &Value) -> Vec { + let mut left = 2048usize; + let mut values = Vec::new(); + for key in ["artist", "title", "album", "genre", "composer"] { + if let Some(value) = tags[key].as_str().filter(|s| !s.trim().is_empty()) { + let prefix = format!("{key}: "); + let mut end = value.len().min(left.saturating_sub(prefix.len())); + while !value.is_char_boundary(end) { + end -= 1; + } + if end == 0 { + break; + } + let text = format!("{prefix}{}", &value[..end]); + left -= text.len(); + values.push(json!({"field":key,"text":text})); + } + } + values +} +fn dot(a: &[f32], b: &[f32]) -> Result { + ensure!( + a.len() == 768 && b.len() == 768 && a.iter().chain(b).all(|x| x.is_finite()), + "Invalid vector" + ); + Ok(a.iter() + .zip(b) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum()) +} +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + ensure!( + args.len() == 7, + "Supply FIXTURE PREPARED MODEL WORKER FFMPEG FFPROBE OUTPUT" + ); + ensure!( + std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"), + "CPU only" + ); + let fixture_path = PathBuf::from(&args[0]); + let fixture_raw = fs::read(&fixture_path)?; + let fixture: Value = serde_json::from_slice(&fixture_raw)?; + let lock: Value = + serde_json::from_slice(&fs::read(fixture_path.with_file_name("labels.lock.json"))?)?; + ensure!( + lock["fixture_sha256"] == sha(&fixture_raw), + "Frozen labels changed" + ); + let plan_raw = fs::read(&args[1])?; + let plan: Value = serde_json::from_slice(&plan_raw)?; + let input_lock: Value = + serde_json::from_slice(&fs::read(fixture_path.with_file_name("inputs.lock.json"))?)?; + ensure!( + input_lock["prepared_plan_sha256"] == sha(&plan_raw) + && input_lock["fixture_sha256"] == sha(&fixture_raw), + "Prepared plan differs from pre-inference input lock" + ); + ensure!( + plan["fixture_sha256"] == sha(&fixture_raw) && plan["fixture"] == fixture, + "Prepared inputs mismatch" + ); + ensure!( + plan["ffmpeg_sha256"] == sha(&fs::read(&args[4])?) + && plan["ffprobe_sha256"] == sha(&fs::read(&args[5])?), + "Decoder changed" + ); + let output = PathBuf::from(&args[6]); + ensure!(!output.exists(), "Preserve existing report"); + let candidates = plan["candidates"] + .as_array() + .context("Candidates missing")?; + let tools = FfmpegTools::new(PathBuf::from(&args[4]), PathBuf::from(&args[5]))?; + let limits = MediaLimits::default(); + let mut model = GemmaEmbedder::load_with_device(&args[2], &args[3], 1, Device::Cpu)?; + ensure!(model.info.device == "cpu", "CPU required"); + let started = Instant::now(); + let mut media: BTreeMap)> = BTreeMap::new(); + let mut metadata = Vec::new(); + let mut decoded = BTreeMap::new(); + for candidate in candidates { + let path = PathBuf::from(candidate["path"].as_str().context("Path missing")?); + ensure!( + candidate["sha256"] == sha(&fs::read(&path)?), + "Candidate bytes changed" + ); + let id = candidate["id"].as_str().context("ID missing")?.to_string(); + if candidate["kind"] == "audio" { + let actual = probe(&tools, &path, &limits)?; + ensure!( + serde_json::to_value(&actual.tags)? == candidate["actual_tags"], + "Actual tags differ" + ); + let window = audio_windows(&actual, &limits)? + .into_iter() + .next() + .context("Audio absent")?; + let pcm = decode_audio(&tools, &path, window, &limits)?; + let bytes: Vec<_> = pcm.iter().flat_map(|s| s.to_le_bytes()).collect(); + let hash = sha(&bytes); + if let Some(expected) = candidate["pcm_sha256"].as_str() { + ensure!(hash == expected, "Prepared PCM changed"); + } + if let Some(old) = decoded.insert(id.clone(), hash.clone()) { + ensure!(old == hash, "Variants changed decoded audio"); + } + if !media.contains_key(&id) { + media.insert(id.clone(),(json!({"asset":id,"kind":"audio","path":path,"start_ms":0,"duration_ms":window.duration_ms}),model.embed_audio_pcm(&pcm)?)); + } + } else if !media.contains_key(&id) { + media.insert( + id.clone(), + ( + json!({"asset":id,"kind":"image","path":path}), + model.embed_images_with_budget(&[&path], 280)?.remove(0), + ), + ); + } + for field in fields(&candidate["actual_tags"]) { + let text = field["text"].as_str().context("Text missing")?; + metadata.push((json!({"asset":id,"kind":"metadata","variant":candidate["variant"],"field":field["field"],"path":path,"text":text}),model.embed_documents(&[text],"File metadata")?.remove(0))); + } + println!("Prepared {} {}", id, candidate["variant"]); + } + let encoding_ms = started.elapsed().as_secs_f64() * 1000.0; + let mut rows = Vec::new(); + for query in fixture["queries"].as_array().context("Queries missing")? { + let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?; + let mut lanes = Vec::new(); + for variant in [ + "audio_only", + "original", + "missing_metadata", + "misleading_metadata", + ] { + let mut ranked: Vec<_> = media + .values() + .map(|(source, encoded)| Ok(json!({"source":source,"score":dot(&vector,encoded)?}))) + .collect::>()?; + for (source, encoded) in &metadata { + if variant != "audio_only" + && (source["variant"] == variant || source["variant"] == "fixed") + { + let score = dot(&vector, encoded)?; + let hit = ranked + .iter_mut() + .find(|h| h["source"]["asset"] == source["asset"]) + .context("Missing candidate")?; + if score > hit["score"].as_f64().context("Score missing")? { + *hit = json!({"source":source,"score":score}); + } + } + } + ranked.sort_by(|a, b| { + b["score"] + .as_f64() + .unwrap() + .total_cmp(&a["score"].as_f64().unwrap()) + }); + lanes.push(json!({"variant":variant,"file_rankings":ranked})); + } + rows.push(json!({"query":query,"lanes":lanes})); + } + for candidate in candidates { + ensure!( + candidate["sha256"] == sha(&fs::read(candidate["path"].as_str().unwrap())?), + "Candidate changed during inference" + ); + } + let report = json!({"fixture_sha256":sha(&fixture_raw),"prepared_sha256":sha(&plan_raw),"worker_sha256":sha(&fs::read(&args[3])?),"worker":model.info,"encoding_ms":encoding_ms,"media_inference_calls":media.len(),"metadata_inference_calls":metadata.len(),"rows":rows,"policy":"File max, real field tags,neutral File metadata title,total2048bytes,no cutoff/weights; all fixed distractors retained; first <=30s audio clips; images280","production_policy_changed":false}); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(output)?; + file.write_all(&serde_json::to_vec_pretty(&report)?)?; + file.sync_all()?; + Ok(()) +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn missing_and_encoder_tags_add_nothing() { + assert!(fields(&json!({"encoder":"violin","artist":" "})).is_empty()); + } + #[test] + fn misleading_tags_are_literal() { + assert_eq!( + fields(&json!({"title":"Wrong violin"}))[0]["text"], + "title: Wrong violin" + ); + } + #[test] + fn total_utf8_budget_is_bounded() { + assert!( + fields(&json!({"title":"ä".repeat(4000)})) + .iter() + .map(|f| f["text"].as_str().unwrap().len()) + .sum::() + <= 2048 + ); + } + #[test] + fn invalid_vectors_fail() { + assert!(dot(&[0.0], &[0.0]).is_err()); + assert!(dot(&vec![f32::NAN; 768], &vec![0.0; 768]).is_err()); + } +} diff --git a/tools/speech-rank/README.md b/tools/speech-rank/README.md index ae5311e..e774e88 100644 --- a/tools/speech-rank/README.md +++ b/tools/speech-rank/README.md @@ -83,3 +83,13 @@ in the embedded ASR evidence and are independently checked; they do not authoriz fabricating or clamping DTW points. Missing/invalid DTW points exclude a window and prevent accepted scoring. Use a separate DTW report/output and retain the failed ordinary-v1 run. The frozen five sources/eight queries remain unchanged. + +The separately frozen `gated-dtw-v1` combination embeds exact manifest bytes +with SHA `d3bf453f21dac03fd029344af2fe700480cf686d2ba5b5d7fcc79134dda1c563`. +The ranker verifies component hashes and the original frozen VAD bytes, rebuilds +all twelve speech windows from source intersections, requires explicit no-speech +status for empty competitors, then independently reconstructs DTW points. All +five sources remain mandatory; dropped windows, fabricated no-speech claims, +tampered policy bytes and invalid points are rejected. This combination leaves +the old gated base and ungated DTW policies unchanged. Structural token tests +provide no new quality result; actual model inference remains a separate gate. diff --git a/tools/speech-rank/src/alignment.rs b/tools/speech-rank/src/alignment.rs index 83fceeb..c6b36e4 100644 --- a/tools/speech-rank/src/alignment.rs +++ b/tools/speech-rank/src/alignment.rs @@ -12,8 +12,50 @@ pub const DTW_POLICY: &str = pub const DTW_POLICY_SHA: &str = "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5"; pub const DTW_CONTRACT_SHA: &str = "61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f"; +pub const COMBINATION_POLICY: &str = + "gated-dtw-v1;frozen-silero-source-intersections;dtw-token-sentences-v1;all-five-sources"; +pub const COMBINATION_SHA: &str = + "d3bf453f21dac03fd029344af2fe700480cf686d2ba5b5d7fcc79134dda1c563"; + +fn verify_combination(asr: &Value) -> Result<()> { + use sha2::{Digest, Sha256}; + let bytes = asr["combination_policy_utf8"] + .as_str() + .context("Missing original combination policy")? + .as_bytes(); + ensure!( + bytes.len() <= 16384 + && format!("{:x}", Sha256::digest(bytes)) == COMBINATION_SHA + && asr["combination_policy_sha256"] == COMBINATION_SHA + && asr["combination_policy"] == COMBINATION_POLICY, + "Frozen combined policy changed" + ); + let plan: Value = serde_json::from_slice(bytes)?; + ensure!( + plan["policy"] == COMBINATION_POLICY + && plan["dtw_policy_sha256"] == DTW_POLICY_SHA + && plan["dtw_contract_sha256"] == DTW_CONTRACT_SHA + && plan["ordinary_contract_sha256"] == CONTRACT_SHA + && plan["asr_worker_sha256"] == asr["worker_sha256"] + && plan["vad_report_sha256"] == asr["vad_report_sha256"] + && plan["sources"] == json!(super::ASSETS), + "Combined component provenance differs" + ); + Ok(()) +} pub fn enabled(asr: &Value) -> Result { + let combined = asr["policy"] == super::gated::POLICY && asr["alignment_policy"] == DTW_POLICY; + if combined { + verify_combination(asr)?; + } else { + ensure!( + asr.get("combination_policy").is_none() + && asr.get("combination_policy_sha256").is_none() + && asr.get("combination_policy_utf8").is_none(), + "Combined policy outside declared gated DTW mode" + ); + } if asr.get("alignment_policy").is_none() { ensure!( asr.get("alignment_contract_sha256").is_none() @@ -32,7 +74,7 @@ pub fn enabled(asr: &Value) -> Result { && asr["candidate"] == "base" && asr["worker_sha256"] == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8" - && asr["policy"] == super::POLICY, + && (asr["policy"] == super::POLICY || combined), "Unknown token alignment provenance" ); Ok(true) diff --git a/tools/speech-rank/src/main.rs b/tools/speech-rank/src/main.rs index 1f4097f..5a17429 100644 --- a/tools/speech-rank/src/main.rs +++ b/tools/speech-rank/src/main.rs @@ -374,6 +374,10 @@ fn main() -> Result<()> { report["alignment_policy"] = asr["alignment_policy"].clone(); report["alignment_policy_sha256"] = asr["alignment_policy_sha256"].clone(); report["alignment_contract_sha256"] = asr["alignment_contract_sha256"].clone(); + if asr.get("combination_policy").is_some() { + report["combination_policy"] = asr["combination_policy"].clone(); + report["combination_policy_sha256"] = asr["combination_policy_sha256"].clone(); + } } report["phase"] = json!("document_encoding"); checkpoint(&output, &report)?; @@ -487,6 +491,68 @@ mod tests { .is_err()); } + #[test] + fn frozen_gated_dtw_pool_requires_all_sources_windows_and_policy_bytes() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let path = root.join(".searchi/speech/gated-base-results.json"); + if !path.exists() { + return; + } + let fixture: Value = + serde_json::from_slice(&fs::read(root.join("fixtures/media/fixture.json")).unwrap()) + .unwrap(); + let mut report: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let plan = include_str!("../../speech-vad/gated-dtw-policy.json"); + report["combination_policy_utf8"] = json!(plan); + report["combination_policy"] = json!(alignment::COMBINATION_POLICY); + report["combination_policy_sha256"] = json!(alignment::COMBINATION_SHA); + report["alignment_policy"] = json!(alignment::DTW_POLICY); + report["alignment_policy_sha256"] = json!(alignment::DTW_POLICY_SHA); + report["alignment_contract_sha256"] = json!(alignment::DTW_CONTRACT_SHA); + // Explicitly synthetic token points test the coverage/protocol path, + // not actual alignment or quality. Original frozen VAD windows unchanged. + for row in report["rows"].as_array_mut().unwrap() { + for window in row["windows"].as_array_mut().unwrap() { + for segment in window["actual_whisper_json"]["transcription"] + .as_array_mut() + .unwrap() + { + segment["tokens"] = json!([{"id":100,"text":segment["text"],"offsets":segment["offsets"],"t_dtw":number(&segment["offsets"]["from"]).unwrap()/10}]); + } + window["aligned_segments"] = json!(alignment::reconstruct_mode( + &window["actual_whisper_json"], + number(&window["offset_ms"]).unwrap(), + number(&window["duration_ms"]).unwrap(), + true + ) + .unwrap()); + } + } + assert!(eligible(&report, &fixture).unwrap().1.is_empty()); + let mut changed = report.clone(); + changed["combination_policy_utf8"] = json!(format!("{plan}\n")); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report.clone(); + changed["rows"].as_array_mut().unwrap().pop(); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report.clone(); + changed["rows"][0]["windows"].as_array_mut().unwrap().pop(); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report.clone(); + let no_speech = changed["rows"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|row| row["status"] == "no_speech") + .unwrap(); + no_speech["asr_not_run"] = json!(false); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report; + changed["rows"][0]["windows"][0]["actual_whisper_json"]["transcription"][0]["tokens"][0] + ["t_dtw"] = json!(-1); + assert!(eligible(&changed, &fixture).is_err()); + } + #[test] fn aligned_pool_preserves_all_competitors_and_marks_real_failures_ineligible() { let hash = "a".repeat(64); diff --git a/tools/speech-vad/README.md b/tools/speech-vad/README.md index 7fe88fd..e37b8dc 100644 --- a/tools/speech-vad/README.md +++ b/tools/speech-vad/README.md @@ -36,3 +36,36 @@ VAD report. It preserves all five competitors, no-speech status, original source intervals and raw ASR logs. `compare.py BASELINE GATED` reports component timing and segment counts only; it rejects incomplete or invalid pools and does not score retrieval quality. See the protocol document for actual evidence. +# Separate frozen VAD + DTW comparison + +`gated-dtw-policy.json` combines the already frozen Silero source-intersection +decisions with the separately declared base DTW sentence-point contract. +Add `--dtw-sentences` to `gated_asr.py` and select a new output, for example +`.searchi/speech/gated-base-dtw-results-v1.json`. All five source rows remain +mandatory; explicit no-speech rows use unchanged actual frozen VAD evidence and +launch no ASR. Neither relevance labels nor the observed malformed piano output +selects a gate or a window. Original raw Google audio retrieval stays unchanged. +Original raw recognized segments and ordinary token errors remain recorded. +Valid DTW points alone determine the separate aligned candidate times; missing +or invalid points make accepted comparison scoring ineligible. + +The combined manifest pins unchanged VAD report/worker, base worker, DTW policy, +DTW helper and ordinary helper hashes. It preserves the old gated policy in +`policy`, with separate `alignment_policy` and `combination_policy` provenance. +Rankers must independently validate both policies and reconstruct every frozen +VAD window and DTW point before inference/scoring. Existing gated base output +is not overwritten. A combination plan or structural tests establish no new +speech recognition or timestamp quality. + +Run only after the coordinator freezes the plan and grants the exclusive CPU +slot. Use `--no-gpu --threads 1 --processors 1 --dtw base --no-flash-attn`; the +harness adds these explicitly. The twelve existing speech windows remain +bounded to 30 seconds, 4 MiB raw output and 600 seconds each. Observed gated +base processing previously took about 65 seconds including VAD cost; DTW cost +and memory have not been measured. Native DTW workspace defaults to an additional +128 MiB, which is not a total memory ceiling. Strict parse failures retain bounded +raw bytes and atomically persist incomplete status, then stop instead of skipping +a competitor. No full rerun has established this combined policy. +DTW native logs are retained byte-exact as bounded base64 plus SHA so malformed +recognized bytes in diagnostic stderr cannot cause lossy decoding. Recognized +JSON remains strict UTF-8 and cannot be repaired by this log representation. diff --git a/tools/speech-vad/gated-dtw-policy.json b/tools/speech-vad/gated-dtw-policy.json new file mode 100644 index 0000000..01b5a24 --- /dev/null +++ b/tools/speech-vad/gated-dtw-policy.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "policy": "gated-dtw-v1;frozen-silero-source-intersections;dtw-token-sentences-v1;all-five-sources", + "vad_report_sha256": "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df", + "vad_worker_sha256": "28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19", + "asr_worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8", + "dtw_policy_sha256": "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5", + "dtw_contract_sha256": "61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f", + "ordinary_contract_sha256": "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520", + "sources": ["clip04", "music01", "music02", "sound01", "sound02"], + "rules": "Unchanged frozen VAD decisions/windows; explicit no-speech rows launch no ASR; retained original raw audio retrieval is unchanged; no labels select speech intervals or suppress sources", + "production_default": false +} diff --git a/tools/speech-vad/gated_asr.py b/tools/speech-vad/gated_asr.py index ef1246f..85d266a 100644 --- a/tools/speech-vad/gated_asr.py +++ b/tools/speech-vad/gated_asr.py @@ -5,6 +5,8 @@ import json import math from pathlib import Path import tempfile +import sys +import base64 spec = importlib.util.spec_from_file_location("gated_speech_contract", Path(__file__).parents[1] / "speech-eval/run.py") speech = importlib.util.module_from_spec(spec) @@ -16,6 +18,22 @@ VAD_SHA = "093be2005b57fe16a8f85161cc9da8966e70e2050cf19fa4122531e304dd70df" VAD_WORKER_SHA = "28746d770ed470d13475d9474fcd37f28c38c39f27da09219bf9a15eef2adb19" WHISPER_SHA = "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8" POLICY = "gated-vad-v1;source-intersections;integer-ms-tail-remainder-recorded;contiguous30s;independent-process;auto-language;segment-times" +HARNESS_SHA_AT_START = speech.sha256(Path(__file__)) + + +def dtw_contract(): + plan_path = Path(__file__).with_name("gated-dtw-policy.json") + plan = json.loads(plan_path.read_text(encoding="utf-8")) + directory = Path(__file__).parents[1] / "speech-eval" + for filename, key in [("dtw-policy.json", "dtw_policy_sha256"), + ("dtw_alignment.py", "dtw_contract_sha256"), + ("token_alignment.py", "ordinary_contract_sha256")]: + if speech.sha256(directory / filename) != plan[key]: + raise ValueError("Frozen gated DTW contract mismatch") + if (plan["vad_report_sha256"] != VAD_SHA or plan["asr_worker_sha256"] != WHISPER_SHA + or plan["vad_worker_sha256"] != VAD_WORKER_SHA or plan["sources"] != list(speech.ASSETS)): + raise ValueError("Frozen gated DTW provenance mismatch") + return plan, speech.sha256(plan_path) def interval_windows(intervals, samples, origin): @@ -72,7 +90,13 @@ def main(): for option in ("whisper", "model", "ffmpeg", "ffprobe", "vad-report", "scratch-parent", "output"): parser.add_argument("--" + option, type=Path, required=True) parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json")) + parser.add_argument("--dtw-sentences", action="store_true", help="Separate frozen VAD+DTW comparison") args = parser.parse_args() + if args.dtw_sentences: + combined, combined_sha = dtw_contract() + sys.path.insert(0, str(Path(__file__).parents[1] / "speech-eval")) + from dtw_alignment import aligned_segments, POLICY as DTW_POLICY + from token_alignment import aligned_segments as ordinary_segments if args.output.exists(): raise ValueError("Never overwrite experiment evidence") if speech.sha256(args.vad_report) != VAD_SHA or speech.sha256(args.fixture) != speech.FIXTURE_SHA: @@ -96,17 +120,23 @@ def main(): "worker_sha256": WHISPER_SHA, "backend": "forced CPU", "threads": 1, "policy": POLICY, "vad_report_sha256": VAD_SHA, "actual_vad_report": document, "vad_report_utf8": vad_bytes.decode("utf-8"), - "harness_sha256": speech.sha256(Path(__file__)), + "harness_sha256": HARNESS_SHA_AT_START, "speech_contract_sha256": speech.sha256(Path(spec.origin)), "ffmpeg_sha256": document["ffmpeg_sha256"], "ffprobe_sha256": document["ffprobe_sha256"], "scope": "Gated ASR diagnostic; no product or word-alignment claim", "rows": []} + if args.dtw_sentences: + report.update(alignment_policy=DTW_POLICY, + alignment_policy_sha256=combined["dtw_policy_sha256"], + alignment_contract_sha256=combined["dtw_contract_sha256"], + combination_policy=combined["policy"], combination_policy_sha256=combined_sha, + combination_policy_utf8=Path(__file__).with_name("gated-dtw-policy.json").read_bytes().decode("utf-8")) args.output.parent.mkdir(parents=True, exist_ok=True) args.scratch_parent.mkdir(parents=True, exist_ok=True) with args.output.open("x", encoding="utf-8") as stream: json.dump(report, stream) def checkpoint(): - args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + speech.checkpoint_report(args.output, report) with tempfile.TemporaryDirectory(prefix="gated-asr-", dir=args.scratch_parent) as directory: scratch = Path(directory) @@ -137,15 +167,44 @@ def main(): frames = speech.decoded_frames(wav, length) asr_ms = speech.run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav), - "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, speech.MAX_OUTPUT)], 600) - raw = json.loads(output.read_text(encoding="utf-8")) + "--output-json-full" if args.dtw_sentences else "--output-json", + *(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []), + "--output-file", str(prefix)], scratch, "asr", [(output, speech.MAX_OUTPUT)], 600) + try: + raw = speech.parse_asr_output(output, args.scratch_parent) + except speech.AsrOutputError as error: + row["windows"].append({**planned, "decoded_frames": frames, + "error_stage": "asr_output_parse", "validation_error": str(error), + "actual_output_failure": error.failure, "segments": []}) + report.update(quality_failed=True, comparison_eligible=False, + failure={"asset":asset["id"], "offset_ms":offset, **error.failure}) + checkpoint() + raise window = speech.recognized_window(raw, offset, length, decode_ms, asr_ms) window.update(planned) window["decoded_frames"] = frames - window["actual_asr_log"] = (scratch / "asr.log").read_text(encoding="utf-8") + if args.dtw_sentences: + log_bytes = (scratch / "asr.log").read_bytes() + if len(log_bytes) > speech.MAX_OUTPUT: + raise ValueError("ASR log exceeds bound") + window["actual_asr_log_base64"] = base64.b64encode(log_bytes).decode("ascii") + window["actual_asr_log_sha256"] = speech.sha256(scratch / "asr.log") + else: + window["actual_asr_log"] = (scratch / "asr.log").read_text(encoding="utf-8") + if args.dtw_sentences: + try: + ordinary_segments(raw, offset, length) + except (ValueError, KeyError, TypeError) as error: + window["ordinary_alignment_validation_error"] = str(error) + try: + window["aligned_segments"] = aligned_segments(raw, offset, length) + except (ValueError, KeyError, TypeError) as error: + window["alignment_validation_error"] = str(error) row["windows"].append(window) if "validation_error" in window: report["validation_failures"].append({"asset": asset["id"], "offset_ms": offset, "error": window["validation_error"]}) + if "alignment_validation_error" in window: + report["validation_failures"].append({"asset": asset["id"], "offset_ms": offset, "error": window["alignment_validation_error"]}) checkpoint() if source.stat().st_size != asset["bytes"] or speech.sha256(source) != asset["sha256"]: raise ValueError("Source changed during recognition") @@ -161,6 +220,8 @@ def main(): or any(speech.sha256(getattr(args, name)) != document[name + "_sha256"] for name in ("ffmpeg", "ffprobe"))): raise ValueError("Experiment tools/provenance changed during recognition") report["complete"] = True + if args.dtw_sentences and dtw_contract()[1] != combined_sha: + raise ValueError("Gated DTW contracts changed during recognition") report["quality_failed"] = bool(report["validation_failures"]) report["comparison_eligible"] = not report["quality_failed"] checkpoint() diff --git a/tools/speech-vad/test_gated_asr.py b/tools/speech-vad/test_gated_asr.py index 25dca36..18493b5 100644 --- a/tools/speech-vad/test_gated_asr.py +++ b/tools/speech-vad/test_gated_asr.py @@ -3,11 +3,20 @@ import copy import json from pathlib import Path import unittest +from unittest.mock import patch import gated_asr as gated class GatedPlanning(unittest.TestCase): + def test_frozen_dtw_combination_preserves_sources_and_component_hashes(self): + plan, digest = gated.dtw_contract() + self.assertEqual(plan["sources"], list(gated.speech.ASSETS)) + self.assertEqual(plan["vad_report_sha256"], gated.VAD_SHA) + self.assertEqual(len(digest), 64) + with patch.object(gated.speech, "sha256", return_value="changed"): + with self.assertRaises(ValueError): + gated.dtw_contract() def test_absolute_offset_and_contiguous_split(self): interval = gated.vad.source_intersection([10, 6110], 1000000, 5125) windows = gated.interval_windows([interval], 1000000, 5125)