karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Preserve exact frozen music and reranker manifest bytes across checkouts

Browse Source
This commit is contained in:
karl committed 2026-10-13 19:01:08 +02:00
1 parent ec73d453e3
commit 512ec4ea9f
9 files changed
+1614 -1609
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitattributes
+5
View File
@@ -6,3 +6,8 @@
tools/embeddinggemma2/*.patch -whitespace
# Frozen quality snapshots must retain exact bytes across operating systems.
fixtures/mixed/sources/** -text
# Frozen diagnostic manifests/labels retain the exact bytes used in recorded runs.
fixtures/music-validation-v1/*.json -text
tools/rerank-eval/*-plan.json -text
tools/rerank-eval/*-manifest.json -text
fixtures/music-validation-v1/evaluation-summary.json
+844 -844
View File
File diff suppressed because it is too large. Load diff
fixtures/music-validation-v1/fixture.json
+268 -268
View File
@@ -1,268 +1,268 @@
{
"version": 1,
"frozen_before_inference": true,
"assets": [
{
"id": "unseen01",
"filename": "AcousticGuitarSample.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:AcousticGuitarSample.ogg",
"bytes": 889069,
"sha1": "d622d15b17b5c101631dd7707a57cf24b9c474da",
"license": "Public domain",
"license_url": null,
"credit_html": "<a href=\"//commons.wikimedia.org/w/index.php?title=User:RyGuy&amp;action=edit&amp;redlink=1\" class=\"new\" title=\"User:RyGuy (page does not exist)\">RyGuy</a>",
"publisher_description_html": "Sample of the sounds which can be made by an acoustic guitar.",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen02",
"filename": "Cello strings.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Cello_strings.ogg",
"bytes": 296474,
"sha1": "8c965bc8679ac6b84b6abeaa086311892d590cae",
"license": "CC BY-SA 3.0",
"license_url": "http://creativecommons.org/licenses/by-sa/3.0/",
"credit_html": null,
"publisher_description_html": null,
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen03",
"filename": "Drum - Cadence A.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Drum_-_Cadence_A.ogg",
"bytes": 1574547,
"sha1": "19d4fbf201e4990d9b2be9c454b209f36af86700",
"license": "Public domain",
"license_url": null,
"credit_html": "Unknown author<span style=\"display: none;\">Unknown author</span>",
"publisher_description_html": "<a href=\"https://en.wikipedia.org/wiki/Drum_cadence\" class=\"extiw\" title=\"w:Drum cadence\">Drum Cadence</a> A performed by the <a href=\"https://en.wikipedia.org/wiki/United_States_Navy_Band\" class=\"extiw\" title=\"wikipedia:United States Navy Band\">United States Navy Band</a>",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen04",
"filename": "Bucket mute trumpet demo.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Bucket_mute_trumpet_demo.ogg",
"bytes": 228972,
"sha1": "37ebe58d5d3eb6051b066ed31bcc66240518b86c",
"license": "CC BY-SA 4.0",
"license_url": "https://creativecommons.org/licenses/by-sa/4.0",
"credit_html": "<a href=\"//commons.wikimedia.org/wiki/User:Ovinus_Real\" class=\"mw-redirect\" title=\"User:Ovinus Real\">Ovinus Real</a>",
"publisher_description_html": "I play the concert notes F3, Bb3, Eb4, Ab4, Db5, Gb5, B5 in succession on a Bb trumpet with bucket mute inserted. I then play a short jazz-ish excerpt (that I had previously improvised) to demonstrate the character of the tone. The sound was amplified but not processed in any other way.",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen05",
"filename": "Afteryouvegone-first8bars.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Afteryouvegone-first8bars.ogg",
"bytes": 300001,
"sha1": "92eee317927ff5cdc5614fe848350cfde2419df3",
"license": "CC BY-SA 3.0",
"license_url": "https://creativecommons.org/licenses/by-sa/3.0",
"credit_html": "<a href=\"//commons.wikimedia.org/wiki/User:Julien_Grandgagnage\" class=\"mw-redirect\" title=\"User:Julien Grandgagnage\">Julien Grandgagnage</a>",
"publisher_description_html": "After You've Gone - first 8 bars on tenor saxophone",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen06",
"filename": "Aintshesweet8bars.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Aintshesweet8bars.ogg",
"bytes": 250846,
"sha1": "3fb787158e3250e8a30f99d82fd9ee0f4743511c",
"license": "CC BY-SA 3.0",
"license_url": "https://creativecommons.org/licenses/by-sa/3.0",
"credit_html": "<a href=\"//commons.wikimedia.org/wiki/User:Julien_Grandgagnage\" class=\"mw-redirect\" title=\"User:Julien Grandgagnage\">Julien Grandgagnage</a>",
"publisher_description_html": "Ain't She Sweet, first 8 bars on soprano saxophone",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
}
],
"queries": [
{
"id": "unseen01-en",
"text": "a recording of acoustic guitar",
"language": "en",
"expected": [
"unseen01"
],
"category": "instrument"
},
{
"id": "unseen01-de",
"text": "eine Aufnahme mit akustische Gitarre",
"language": "de",
"expected": [
"unseen01"
],
"category": "instrument"
},
{
"id": "unseen02-en",
"text": "a recording of cello strings",
"language": "en",
"expected": [
"unseen02"
],
"category": "instrument"
},
{
"id": "unseen02-de",
"text": "eine Aufnahme mit Cellosaiten",
"language": "de",
"expected": [
"unseen02"
],
"category": "instrument"
},
{
"id": "unseen03-en",
"text": "a recording of military drum cadence",
"language": "en",
"expected": [
"unseen03"
],
"category": "instrument"
},
{
"id": "unseen03-de",
"text": "eine Aufnahme mit militärische Trommelkadenz",
"language": "de",
"expected": [
"unseen03"
],
"category": "instrument"
},
{
"id": "unseen04-en",
"text": "a recording of trumpet with a bucket mute",
"language": "en",
"expected": [
"unseen04"
],
"category": "instrument"
},
{
"id": "unseen04-de",
"text": "eine Aufnahme mit Trompete mit Bucket-Dämpfer",
"language": "de",
"expected": [
"unseen04"
],
"category": "instrument"
},
{
"id": "unseen05-en",
"text": "a recording of tenor saxophone melody",
"language": "en",
"expected": [
"unseen05"
],
"category": "instrument"
},
{
"id": "unseen05-de",
"text": "eine Aufnahme mit Tenorsaxophon-Melodie",
"language": "de",
"expected": [
"unseen05"
],
"category": "instrument"
},
{
"id": "unseen06-en",
"text": "a recording of soprano saxophone melody",
"language": "en",
"expected": [
"unseen06"
],
"category": "instrument"
},
{
"id": "unseen06-de",
"text": "eine Aufnahme mit Sopransaxophon-Melodie",
"language": "de",
"expected": [
"unseen06"
],
"category": "instrument"
},
{
"id": "sax-broad-en",
"text": "a saxophone playing a melody",
"language": "en",
"expected": [
"unseen05",
"unseen06"
],
"category": "related-positive"
},
{
"id": "sax-broad-de",
"text": "eine Saxophonmelodie",
"language": "de",
"expected": [
"unseen05",
"unseen06"
],
"category": "related-positive"
},
{
"id": "absent-bagpipe-en",
"text": "a bagpipe performance",
"language": "en",
"expected": [],
"category": "absent-instrument"
},
{
"id": "absent-bagpipe-de",
"text": "eine Dudelsackaufnahme",
"language": "de",
"expected": [],
"category": "absent-instrument"
},
{
"id": "absent-artist-en",
"text": "Adele singing Hello",
"language": "en",
"expected": [],
"category": "absent-identity"
},
{
"id": "absent-content-en",
"text": "a person explaining quantum entanglement",
"language": "en",
"expected": [],
"category": "absent-content"
}
],
"variants": [
"original",
"missing_metadata",
"misleading_metadata"
],
"misleading_policy": "Whitelisted actual tags cyclically transferred from the next asset; no invented descriptive tags. All clips encode first <=30seconds as lossless FLAC mono16kHz, metadata stripped or replaced. Verify exact decoded PCM SHA256 equality across variants before reusing audio vectors. Report controlled corruption separately.",
"candidate_policy": "New six sources plus optional predeclared legacy image/audio/video distractors, fixed before inference; never treat related plausible music as absent. Broad sax labels credit both tenor and soprano. Source title is not instrument evidence for the raw-audio lane.",
"metadata_policy": "Whitelisted artist/title/album/genre/composer fields separately, total2048 UTF8bytes, neutral File metadata title, file maximum over real metadata and audio, noweights/nocutoff.",
"legacy_distractors": [
"image01",
"image02",
"sound01",
"sound02"
]
}
{
"version": 1,
"frozen_before_inference": true,
"assets": [
{
"id": "unseen01",
"filename": "AcousticGuitarSample.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:AcousticGuitarSample.ogg",
"bytes": 889069,
"sha1": "d622d15b17b5c101631dd7707a57cf24b9c474da",
"license": "Public domain",
"license_url": null,
"credit_html": "<a href=\"//commons.wikimedia.org/w/index.php?title=User:RyGuy&amp;action=edit&amp;redlink=1\" class=\"new\" title=\"User:RyGuy (page does not exist)\">RyGuy</a>",
"publisher_description_html": "Sample of the sounds which can be made by an acoustic guitar.",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen02",
"filename": "Cello strings.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Cello_strings.ogg",
"bytes": 296474,
"sha1": "8c965bc8679ac6b84b6abeaa086311892d590cae",
"license": "CC BY-SA 3.0",
"license_url": "http://creativecommons.org/licenses/by-sa/3.0/",
"credit_html": null,
"publisher_description_html": null,
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen03",
"filename": "Drum - Cadence A.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Drum_-_Cadence_A.ogg",
"bytes": 1574547,
"sha1": "19d4fbf201e4990d9b2be9c454b209f36af86700",
"license": "Public domain",
"license_url": null,
"credit_html": "Unknown author<span style=\"display: none;\">Unknown author</span>",
"publisher_description_html": "<a href=\"https://en.wikipedia.org/wiki/Drum_cadence\" class=\"extiw\" title=\"w:Drum cadence\">Drum Cadence</a> A performed by the <a href=\"https://en.wikipedia.org/wiki/United_States_Navy_Band\" class=\"extiw\" title=\"wikipedia:United States Navy Band\">United States Navy Band</a>",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen04",
"filename": "Bucket mute trumpet demo.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Bucket_mute_trumpet_demo.ogg",
"bytes": 228972,
"sha1": "37ebe58d5d3eb6051b066ed31bcc66240518b86c",
"license": "CC BY-SA 4.0",
"license_url": "https://creativecommons.org/licenses/by-sa/4.0",
"credit_html": "<a href=\"//commons.wikimedia.org/wiki/User:Ovinus_Real\" class=\"mw-redirect\" title=\"User:Ovinus Real\">Ovinus Real</a>",
"publisher_description_html": "I play the concert notes F3, Bb3, Eb4, Ab4, Db5, Gb5, B5 in succession on a Bb trumpet with bucket mute inserted. I then play a short jazz-ish excerpt (that I had previously improvised) to demonstrate the character of the tone. The sound was amplified but not processed in any other way.",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen05",
"filename": "Afteryouvegone-first8bars.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Afteryouvegone-first8bars.ogg",
"bytes": 300001,
"sha1": "92eee317927ff5cdc5614fe848350cfde2419df3",
"license": "CC BY-SA 3.0",
"license_url": "https://creativecommons.org/licenses/by-sa/3.0",
"credit_html": "<a href=\"//commons.wikimedia.org/wiki/User:Julien_Grandgagnage\" class=\"mw-redirect\" title=\"User:Julien Grandgagnage\">Julien Grandgagnage</a>",
"publisher_description_html": "After You've Gone - first 8 bars on tenor saxophone",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
},
{
"id": "unseen06",
"filename": "Aintshesweet8bars.ogg",
"url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg",
"source_page": "https://commons.wikimedia.org/wiki/File:Aintshesweet8bars.ogg",
"bytes": 250846,
"sha1": "3fb787158e3250e8a30f99d82fd9ee0f4743511c",
"license": "CC BY-SA 3.0",
"license_url": "https://creativecommons.org/licenses/by-sa/3.0",
"credit_html": "<a href=\"//commons.wikimedia.org/wiki/User:Julien_Grandgagnage\" class=\"mw-redirect\" title=\"User:Julien Grandgagnage\">Julien Grandgagnage</a>",
"publisher_description_html": "Ain't She Sweet, first 8 bars on soprano saxophone",
"clip_start_ms": 0,
"clip_duration_ms": 30000,
"label_basis": "Publisher description only; human listening and second annotator unexecuted."
}
],
"queries": [
{
"id": "unseen01-en",
"text": "a recording of acoustic guitar",
"language": "en",
"expected": [
"unseen01"
],
"category": "instrument"
},
{
"id": "unseen01-de",
"text": "eine Aufnahme mit akustische Gitarre",
"language": "de",
"expected": [
"unseen01"
],
"category": "instrument"
},
{
"id": "unseen02-en",
"text": "a recording of cello strings",
"language": "en",
"expected": [
"unseen02"
],
"category": "instrument"
},
{
"id": "unseen02-de",
"text": "eine Aufnahme mit Cellosaiten",
"language": "de",
"expected": [
"unseen02"
],
"category": "instrument"
},
{
"id": "unseen03-en",
"text": "a recording of military drum cadence",
"language": "en",
"expected": [
"unseen03"
],
"category": "instrument"
},
{
"id": "unseen03-de",
"text": "eine Aufnahme mit militärische Trommelkadenz",
"language": "de",
"expected": [
"unseen03"
],
"category": "instrument"
},
{
"id": "unseen04-en",
"text": "a recording of trumpet with a bucket mute",
"language": "en",
"expected": [
"unseen04"
],
"category": "instrument"
},
{
"id": "unseen04-de",
"text": "eine Aufnahme mit Trompete mit Bucket-Dämpfer",
"language": "de",
"expected": [
"unseen04"
],
"category": "instrument"
},
{
"id": "unseen05-en",
"text": "a recording of tenor saxophone melody",
"language": "en",
"expected": [
"unseen05"
],
"category": "instrument"
},
{
"id": "unseen05-de",
"text": "eine Aufnahme mit Tenorsaxophon-Melodie",
"language": "de",
"expected": [
"unseen05"
],
"category": "instrument"
},
{
"id": "unseen06-en",
"text": "a recording of soprano saxophone melody",
"language": "en",
"expected": [
"unseen06"
],
"category": "instrument"
},
{
"id": "unseen06-de",
"text": "eine Aufnahme mit Sopransaxophon-Melodie",
"language": "de",
"expected": [
"unseen06"
],
"category": "instrument"
},
{
"id": "sax-broad-en",
"text": "a saxophone playing a melody",
"language": "en",
"expected": [
"unseen05",
"unseen06"
],
"category": "related-positive"
},
{
"id": "sax-broad-de",
"text": "eine Saxophonmelodie",
"language": "de",
"expected": [
"unseen05",
"unseen06"
],
"category": "related-positive"
},
{
"id": "absent-bagpipe-en",
"text": "a bagpipe performance",
"language": "en",
"expected": [],
"category": "absent-instrument"
},
{
"id": "absent-bagpipe-de",
"text": "eine Dudelsackaufnahme",
"language": "de",
"expected": [],
"category": "absent-instrument"
},
{
"id": "absent-artist-en",
"text": "Adele singing Hello",
"language": "en",
"expected": [],
"category": "absent-identity"
},
{
"id": "absent-content-en",
"text": "a person explaining quantum entanglement",
"language": "en",
"expected": [],
"category": "absent-content"
}
],
"variants": [
"original",
"missing_metadata",
"misleading_metadata"
],
"misleading_policy": "Whitelisted actual tags cyclically transferred from the next asset; no invented descriptive tags. All clips encode first <=30seconds as lossless FLAC mono16kHz, metadata stripped or replaced. Verify exact decoded PCM SHA256 equality across variants before reusing audio vectors. Report controlled corruption separately.",
"candidate_policy": "New six sources plus optional predeclared legacy image/audio/video distractors, fixed before inference; never treat related plausible music as absent. Broad sax labels credit both tenor and soprano. Source title is not instrument evidence for the raw-audio lane.",
"metadata_policy": "Whitelisted artist/title/album/genre/composer fields separately, total2048 UTF8bytes, neutral File metadata title, file maximum over real metadata and audio, noweights/nocutoff.",
"legacy_distractors": [
"image01",
"image02",
"sound01",
"sound02"
]
}
fixtures/music-validation-v1/inputs.lock.json
+303 -303
View File
@@ -1,303 +1,303 @@
{
"frozen_before_inference": true,
"fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577",
"prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78",
"sources": [
{
"id": "unseen01",
"sha256": "1c604cce5299b2f9579538810deb328d9c47607f098f5885ff1daa7cf34b637a",
"bytes": 889069,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg",
"actual_tags": {
"artist": "Ryan"
}
},
{
"id": "unseen02",
"sha256": "f62b0dabedd570f3abd9fc9e9a3a468375bb77ddd7f1ad02c203c8d5394bf219",
"bytes": 296474,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg?download=1",
"actual_tags": {
"title": "cello strings",
"date": "06/10/2005"
}
},
{
"id": "unseen03",
"sha256": "5083f07730c6ffb1cd2c3b5cc1a43714a0cfda04fe913935e83d1048466562e6",
"bytes": 1574547,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg?download=1",
"actual_tags": {}
},
{
"id": "unseen04",
"sha256": "7fc61102e3050c5fe01000b1b02f839e725b305333505075916faeea78844caa",
"bytes": 228972,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg?download=1",
"actual_tags": {
"artist": "Ovinus Real",
"comments": "First played a series of ascending fourths from concert F3 to B5, then replayed the improvised excerpt from the harmon mute (without stem) recording. No processing or noise removal done.",
"date": "2020",
"title": "Bucket mute"
}
},
{
"id": "unseen05",
"sha256": "0570623af1490ed81b86c5a95f5b95eb61b9d941bb0116e4271e61a07d2dc5ba",
"bytes": 300001,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg?download=1",
"actual_tags": {
"title": "After you've gone - first 8 bars on tenor sax",
"date": "2013",
"artist": "Beachcomber"
}
},
{
"id": "unseen06",
"sha256": "8f2035ab8924ee9f0d212de5853cf22eab7457520c1d774413a78601ed1b8bd6",
"bytes": 250846,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg?download=1",
"actual_tags": {
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"date": "2013",
"artist": "Beachcomber"
}
}
],
"containers": [
{
"id": "unseen01",
"variant": "original",
"kind": "audio",
"sha256": "24b14ccc428e96312a96a556e5250b3d4c781bb42e4549f13b43ddc2fb31c087",
"actual_tags": {
"artist": "Ryan",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen01",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "665ddf557d94a65ff98f1d35563657405846577351de08f00b5a104bb1d6f992",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen01",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "bac63053fc9182e17269ec1015ebf2bc74b803abfb4f52c25a0009f145c7065a",
"actual_tags": {
"title": "cello strings",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen02",
"variant": "original",
"kind": "audio",
"sha256": "fa934449cb45743ffa420c412e3b7bc388ae846e76b29573922ab9506496bbfb",
"actual_tags": {
"title": "cello strings",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen02",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen02",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen03",
"variant": "original",
"kind": "audio",
"sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen03",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen03",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "5e108abbbc44fe068b07a5b124894c4738204f511a308c4385960019f5c453a1",
"actual_tags": {
"artist": "Ovinus Real",
"title": "Bucket mute",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen04",
"variant": "original",
"kind": "audio",
"sha256": "fc498bc0e47fc97b8c085cd00266d3e2dbebecfc3021e443b7b19e43dd025dde",
"actual_tags": {
"artist": "Ovinus Real",
"title": "Bucket mute",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen04",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "09b2df6db7953ed67e5ae5059b3adacbec7e5c13a3be6aa32237b19e106e9fac",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen04",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "fdaae3a8636394100b268a4a2ea8b42565be87fcb8fbdaefc9bee29a42d77b8a",
"actual_tags": {
"artist": "Beachcomber",
"title": "After you've gone - first 8 bars on tenor sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen05",
"variant": "original",
"kind": "audio",
"sha256": "caa19f57058fb7a1d7ac715c087a6c8430f9054b7a3c503ab7b191794f805652",
"actual_tags": {
"artist": "Beachcomber",
"title": "After you've gone - first 8 bars on tenor sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen05",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "1e46c0ec9c8831fa041d27c46cba924803a29fe5304bf5776d598db297b0b24f",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen05",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "56217ac0a4a85921e939c7a84fbe864e1d8c9e1d7b48b6ba54d64187370423aa",
"actual_tags": {
"artist": "Beachcomber",
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen06",
"variant": "original",
"kind": "audio",
"sha256": "27b4a28c800ac7b949af7f92d1e483bc4089c98a45a381b52c816254ef3810fc",
"actual_tags": {
"artist": "Beachcomber",
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "unseen06",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "603b253da7ee235284ab8bb9cdac177001093122ce385032b6a2a00d4ffcb1e8",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "unseen06",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "f8b0e7f4261f69fa326370433dfd6519c09e9a63f12fcaf9dbf47fe8cf481fe2",
"actual_tags": {
"artist": "Ryan",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "image01",
"variant": "fixed",
"kind": "image",
"sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed",
"actual_tags": {}
},
{
"id": "image02",
"variant": "fixed",
"kind": "image",
"sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1",
"actual_tags": {}
},
{
"id": "sound01",
"variant": "fixed",
"kind": "audio",
"sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451",
"actual_tags": {}
},
{
"id": "sound02",
"variant": "fixed",
"kind": "audio",
"sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e",
"actual_tags": {
"encoder": "LAME3.98r"
}
}
],
"ffmpeg_sha256": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46",
"ffprobe_sha256": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236"
}
{
"frozen_before_inference": true,
"fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577",
"prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78",
"sources": [
{
"id": "unseen01",
"sha256": "1c604cce5299b2f9579538810deb328d9c47607f098f5885ff1daa7cf34b637a",
"bytes": 889069,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg",
"actual_tags": {
"artist": "Ryan"
}
},
{
"id": "unseen02",
"sha256": "f62b0dabedd570f3abd9fc9e9a3a468375bb77ddd7f1ad02c203c8d5394bf219",
"bytes": 296474,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg?download=1",
"actual_tags": {
"title": "cello strings",
"date": "06/10/2005"
}
},
{
"id": "unseen03",
"sha256": "5083f07730c6ffb1cd2c3b5cc1a43714a0cfda04fe913935e83d1048466562e6",
"bytes": 1574547,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg?download=1",
"actual_tags": {}
},
{
"id": "unseen04",
"sha256": "7fc61102e3050c5fe01000b1b02f839e725b305333505075916faeea78844caa",
"bytes": 228972,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg?download=1",
"actual_tags": {
"artist": "Ovinus Real",
"comments": "First played a series of ascending fourths from concert F3 to B5, then replayed the improvised excerpt from the harmon mute (without stem) recording. No processing or noise removal done.",
"date": "2020",
"title": "Bucket mute"
}
},
{
"id": "unseen05",
"sha256": "0570623af1490ed81b86c5a95f5b95eb61b9d941bb0116e4271e61a07d2dc5ba",
"bytes": 300001,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg?download=1",
"actual_tags": {
"title": "After you've gone - first 8 bars on tenor sax",
"date": "2013",
"artist": "Beachcomber"
}
},
{
"id": "unseen06",
"sha256": "8f2035ab8924ee9f0d212de5853cf22eab7457520c1d774413a78601ed1b8bd6",
"bytes": 250846,
"source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg",
"actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg?download=1",
"actual_tags": {
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"date": "2013",
"artist": "Beachcomber"
}
}
],
"containers": [
{
"id": "unseen01",
"variant": "original",
"kind": "audio",
"sha256": "24b14ccc428e96312a96a556e5250b3d4c781bb42e4549f13b43ddc2fb31c087",
"actual_tags": {
"artist": "Ryan",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen01",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "665ddf557d94a65ff98f1d35563657405846577351de08f00b5a104bb1d6f992",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen01",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "bac63053fc9182e17269ec1015ebf2bc74b803abfb4f52c25a0009f145c7065a",
"actual_tags": {
"title": "cello strings",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7"
},
{
"id": "unseen02",
"variant": "original",
"kind": "audio",
"sha256": "fa934449cb45743ffa420c412e3b7bc388ae846e76b29573922ab9506496bbfb",
"actual_tags": {
"title": "cello strings",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen02",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen02",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a"
},
{
"id": "unseen03",
"variant": "original",
"kind": "audio",
"sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen03",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen03",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "5e108abbbc44fe068b07a5b124894c4738204f511a308c4385960019f5c453a1",
"actual_tags": {
"artist": "Ovinus Real",
"title": "Bucket mute",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932"
},
{
"id": "unseen04",
"variant": "original",
"kind": "audio",
"sha256": "fc498bc0e47fc97b8c085cd00266d3e2dbebecfc3021e443b7b19e43dd025dde",
"actual_tags": {
"artist": "Ovinus Real",
"title": "Bucket mute",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen04",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "09b2df6db7953ed67e5ae5059b3adacbec7e5c13a3be6aa32237b19e106e9fac",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen04",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "fdaae3a8636394100b268a4a2ea8b42565be87fcb8fbdaefc9bee29a42d77b8a",
"actual_tags": {
"artist": "Beachcomber",
"title": "After you've gone - first 8 bars on tenor sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46"
},
{
"id": "unseen05",
"variant": "original",
"kind": "audio",
"sha256": "caa19f57058fb7a1d7ac715c087a6c8430f9054b7a3c503ab7b191794f805652",
"actual_tags": {
"artist": "Beachcomber",
"title": "After you've gone - first 8 bars on tenor sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen05",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "1e46c0ec9c8831fa041d27c46cba924803a29fe5304bf5776d598db297b0b24f",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen05",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "56217ac0a4a85921e939c7a84fbe864e1d8c9e1d7b48b6ba54d64187370423aa",
"actual_tags": {
"artist": "Beachcomber",
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3"
},
{
"id": "unseen06",
"variant": "original",
"kind": "audio",
"sha256": "27b4a28c800ac7b949af7f92d1e483bc4089c98a45a381b52c816254ef3810fc",
"actual_tags": {
"artist": "Beachcomber",
"title": "Ain't She Sweet, first 8 bars on soprano sax",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "unseen06",
"variant": "missing_metadata",
"kind": "audio",
"sha256": "603b253da7ee235284ab8bb9cdac177001093122ce385032b6a2a00d4ffcb1e8",
"actual_tags": {
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "unseen06",
"variant": "misleading_metadata",
"kind": "audio",
"sha256": "f8b0e7f4261f69fa326370433dfd6519c09e9a63f12fcaf9dbf47fe8cf481fe2",
"actual_tags": {
"artist": "Ryan",
"encoder": "Lavf63.1.102"
},
"pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261"
},
{
"id": "image01",
"variant": "fixed",
"kind": "image",
"sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed",
"actual_tags": {}
},
{
"id": "image02",
"variant": "fixed",
"kind": "image",
"sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1",
"actual_tags": {}
},
{
"id": "sound01",
"variant": "fixed",
"kind": "audio",
"sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451",
"actual_tags": {}
},
{
"id": "sound02",
"variant": "fixed",
"kind": "audio",
"sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e",
"actual_tags": {
"encoder": "LAME3.98r"
}
}
],
"ffmpeg_sha256": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46",
"ffprobe_sha256": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236"
}
fixtures/music-validation-v1/labels.lock.json
+6 -6
View File
@@ -1,6 +1,6 @@
{
"fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577",
"frozen_before_inference": true,
"query_count": 18,
"asset_count": 6
}
{
"fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577",
"frozen_before_inference": true,
"query_count": 18,
"asset_count": 6
}
tools/rerank-eval/compatibility-plan.json
+38 -38
View File
@@ -1,38 +1,38 @@
{
"version": 1,
"status": "pre-inference frozen compatibility/cost contract, not relevance calibration",
"cpu_provider": "CPUExecutionProvider",
"intra_op_threads": 1,
"inter_op_threads": 1,
"parallel_execution": false,
"max_pairs": 5,
"max_pair_tokens": 256,
"max_query_tokens": 64,
"warm_iterations": 20,
"parity_absolute_tolerance": 0.001,
"parity_relative_tolerance": 0.001,
"added_warm_p95_target_ms": 200,
"no_suppression": true,
"pairs": [
[
"How do we cancel the supplier partnership?",
"Either party may terminate the supplier contract by giving sixty days written notice."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen."
],
[
"How do we cancel the supplier partnership?",
"Tomatoes require sunshine and consistent watering."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung."
],
[
"Where does the code implement retry backoff?",
"fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }"
]
]
}
{
"version": 1,
"status": "pre-inference frozen compatibility/cost contract, not relevance calibration",
"cpu_provider": "CPUExecutionProvider",
"intra_op_threads": 1,
"inter_op_threads": 1,
"parallel_execution": false,
"max_pairs": 5,
"max_pair_tokens": 256,
"max_query_tokens": 64,
"warm_iterations": 20,
"parity_absolute_tolerance": 0.001,
"parity_relative_tolerance": 0.001,
"added_warm_p95_target_ms": 200,
"no_suppression": true,
"pairs": [
[
"How do we cancel the supplier partnership?",
"Either party may terminate the supplier contract by giving sixty days written notice."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen."
],
[
"How do we cancel the supplier partnership?",
"Tomatoes require sunshine and consistent watering."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung."
],
[
"Where does the code implement retry backoff?",
"fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }"
]
]
}
tools/rerank-eval/model-manifest.json
+50 -50
View File
@@ -1,50 +1,50 @@
{
"model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
"revision": "1427fd652930e4ba29e8149678df786c240d8825",
"purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval",
"license": "Apache-2.0 per pinned publisher card; text from Apache",
"files": [
{
"path": "onnx/model_quint8_avx2.onnx",
"bytes": 118620016,
"sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx"
},
{
"path": "tokenizer.json",
"bytes": 17082660,
"sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json"
},
{
"path": "config.json",
"bytes": 891,
"sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json"
},
{
"path": "tokenizer_config.json",
"bytes": 435,
"sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json"
},
{
"path": "special_tokens_map.json",
"bytes": 239,
"sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json"
},
{
"path": "README.md",
"bytes": 2278,
"sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md"
},
{
"path": "LICENSE-APACHE-2.0.txt",
"bytes": 11358,
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
"url": "https://www.apache.org/licenses/LICENSE-2.0.txt"
}
]
}
{
"model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
"revision": "1427fd652930e4ba29e8149678df786c240d8825",
"purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval",
"license": "Apache-2.0 per pinned publisher card; text from Apache",
"files": [
{
"path": "onnx/model_quint8_avx2.onnx",
"bytes": 118620016,
"sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx"
},
{
"path": "tokenizer.json",
"bytes": 17082660,
"sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json"
},
{
"path": "config.json",
"bytes": 891,
"sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json"
},
{
"path": "tokenizer_config.json",
"bytes": 435,
"sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json"
},
{
"path": "special_tokens_map.json",
"bytes": 239,
"sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json"
},
{
"path": "README.md",
"bytes": 2278,
"sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md"
},
{
"path": "LICENSE-APACHE-2.0.txt",
"bytes": 11358,
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
"url": "https://www.apache.org/licenses/LICENSE-2.0.txt"
}
]
}
tools/rerank-eval/reference-manifest.json
+56 -56
View File
@@ -1,56 +1,56 @@
{
"model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
"revision": "1427fd652930e4ba29e8149678df786c240d8825",
"purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval",
"license": "Apache-2.0 per pinned publisher card; text from Apache",
"files": [
{
"path": "onnx/model_quint8_avx2.onnx",
"bytes": 118620016,
"sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx"
},
{
"path": "tokenizer.json",
"bytes": 17082660,
"sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json"
},
{
"path": "config.json",
"bytes": 891,
"sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json"
},
{
"path": "tokenizer_config.json",
"bytes": 435,
"sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json"
},
{
"path": "special_tokens_map.json",
"bytes": 239,
"sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json"
},
{
"path": "README.md",
"bytes": 2278,
"sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md"
},
{
"path": "LICENSE-APACHE-2.0.txt",
"bytes": 11358,
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
"url": "https://www.apache.org/licenses/LICENSE-2.0.txt"
},
{
"path": "onnx/model.onnx",
"bytes": 470883696,
"sha256": "3e9a03ed1e966f7c5288dd4230e3d6a9bf5e3a170a06f1f4241c5bca12c6487c",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model.onnx"
}
]
}
{
"model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
"revision": "1427fd652930e4ba29e8149678df786c240d8825",
"purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval",
"license": "Apache-2.0 per pinned publisher card; text from Apache",
"files": [
{
"path": "onnx/model_quint8_avx2.onnx",
"bytes": 118620016,
"sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx"
},
{
"path": "tokenizer.json",
"bytes": 17082660,
"sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json"
},
{
"path": "config.json",
"bytes": 891,
"sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json"
},
{
"path": "tokenizer_config.json",
"bytes": 435,
"sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json"
},
{
"path": "special_tokens_map.json",
"bytes": 239,
"sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json"
},
{
"path": "README.md",
"bytes": 2278,
"sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md"
},
{
"path": "LICENSE-APACHE-2.0.txt",
"bytes": 11358,
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
"url": "https://www.apache.org/licenses/LICENSE-2.0.txt"
},
{
"path": "onnx/model.onnx",
"bytes": 470883696,
"sha256": "3e9a03ed1e966f7c5288dd4230e3d6a9bf5e3a170a06f1f4241c5bca12c6487c",
"url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model.onnx"
}
]
}
tools/rerank-eval/reference-plan.json
+44 -44
View File
@@ -1,44 +1,44 @@
{
"version": 2,
"status": "pre-inference F32 reference parity and 256-token cost contract; original INT8 failure unchanged",
"cpu_provider": "CPUExecutionProvider",
"intra_op_threads": 1,
"inter_op_threads": 1,
"parallel_execution": false,
"max_pairs": 6,
"max_pair_tokens": 256,
"max_query_tokens": 64,
"warm_iterations": 20,
"parity_absolute_tolerance": 0.001,
"parity_relative_tolerance": 0.001,
"added_warm_p95_target_ms": 200,
"no_suppression": true,
"pairs": [
[
"How do we cancel the supplier partnership?",
"Either party may terminate the supplier contract by giving sixty days written notice."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen."
],
[
"How do we cancel the supplier partnership?",
"Tomatoes require sunshine and consistent watering."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung."
],
[
"Where does the code implement retry backoff?",
"fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }"
],
[
"supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice ",
"Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplLine truncated
]
],
"reference_model": "onnx/model.onnx",
"comparison_model": "onnx/model_quint8_avx2.onnx"
}
{
"version": 2,
"status": "pre-inference F32 reference parity and 256-token cost contract; original INT8 failure unchanged",
"cpu_provider": "CPUExecutionProvider",
"intra_op_threads": 1,
"inter_op_threads": 1,
"parallel_execution": false,
"max_pairs": 6,
"max_pair_tokens": 256,
"max_query_tokens": 64,
"warm_iterations": 20,
"parity_absolute_tolerance": 0.001,
"parity_relative_tolerance": 0.001,
"added_warm_p95_target_ms": 200,
"no_suppression": true,
"pairs": [
[
"How do we cancel the supplier partnership?",
"Either party may terminate the supplier contract by giving sixty days written notice."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen."
],
[
"How do we cancel the supplier partnership?",
"Tomatoes require sunshine and consistent watering."
],
[
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
"Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung."
],
[
"Where does the code implement retry backoff?",
"fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }"
],
[
"supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice ",
"Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplLine truncated
]
],
"reference_model": "onnx/model.onnx",
"comparison_model": "onnx/model_quint8_avx2.onnx"
}