From 512ec4ea9fcc21bf15190b4a60c82b56319cef77 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 19:01:08 +0200 Subject: [PATCH] Preserve exact frozen music and reranker manifest bytes across checkouts --- .gitattributes | 5 + .../evaluation-summary.json | 1688 ++++++++--------- fixtures/music-validation-v1/fixture.json | 536 +++--- fixtures/music-validation-v1/inputs.lock.json | 606 +++--- fixtures/music-validation-v1/labels.lock.json | 12 +- tools/rerank-eval/compatibility-plan.json | 76 +- tools/rerank-eval/model-manifest.json | 100 +- tools/rerank-eval/reference-manifest.json | 112 +- tools/rerank-eval/reference-plan.json | 88 +- 9 files changed, 1614 insertions(+), 1609 deletions(-) diff --git a/.gitattributes b/.gitattributes index d16e4c4..9169167 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,8 @@ tools/embeddinggemma2/*.patch -whitespace # Frozen quality snapshots must retain exact bytes across operating systems. fixtures/mixed/sources/** -text + +# Frozen diagnostic manifests/labels retain the exact bytes used in recorded runs. +fixtures/music-validation-v1/*.json -text +tools/rerank-eval/*-plan.json -text +tools/rerank-eval/*-manifest.json -text diff --git a/fixtures/music-validation-v1/evaluation-summary.json b/fixtures/music-validation-v1/evaluation-summary.json index ffaa07e..31851c4 100644 --- a/fixtures/music-validation-v1/evaluation-summary.json +++ b/fixtures/music-validation-v1/evaluation-summary.json @@ -1,844 +1,844 @@ -{ - "source_commit": "cbf07b0", - "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", - "prepared_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78", - "raw_report_sha256": "69bcef5ffa58c0f23e5f34641256255bf15f54c836f75fb1aa878b4bd3df1309", - "raw_report_bytes": 267667, - "worker_sha256": "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73", - "worker": { - "code_query_supported": true, - "context_tokens": 2048, - "device": "cpu", - "dimensions": 768, - "media_supported": true, - "model": "google/embeddinggemma-2", - "ready": true, - "role": "indexing", - "runtime": "b11461-no-logits-v1", - "threads": 1 - }, - "encoding_ms": 88089.5138, - "media_inference_calls": 10, - "metadata_inference_calls": 16, - "metrics": { - "audio_only": { - "positive_queries": 14, - "top1": 8, - "recall_at_5": 14, - "mrr": 0.7261904761904762, - "no_answer_queries": 4, - "no_answer_returned_any": 4 - }, - "original": { - "positive_queries": 14, - "top1": 11, - "recall_at_5": 14, - "mrr": 0.869047619047619, - "no_answer_queries": 4, - "no_answer_returned_any": 4 - }, - "missing_metadata": { - "positive_queries": 14, - "top1": 8, - "recall_at_5": 14, - "mrr": 0.7261904761904762, - "no_answer_queries": 4, - "no_answer_returned_any": 4 - }, - "misleading_metadata": { - "positive_queries": 14, - "top1": 5, - "recall_at_5": 14, - "mrr": 0.5702380952380952, - "no_answer_queries": 4, - "no_answer_returned_any": 4 - } - }, - "changes_vs_audio": { - "original": { - "top1_gains_vs_audio": [ - "unseen02-en", - "unseen02-de", - "unseen06-en", - "unseen06-de" - ], - "top1_losses_vs_audio": [ - "unseen01-de" - ] - }, - "missing_metadata": { - "top1_gains_vs_audio": [], - "top1_losses_vs_audio": [] - }, - "misleading_metadata": { - "top1_gains_vs_audio": [], - "top1_losses_vs_audio": [ - "unseen01-de", - "unseen05-en", - "unseen05-de" - ] - } - }, - "rows": [ - { - "id": "unseen01-en", - "expected": [ - "unseen01" - ], - "language": "en", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6707203058852982 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6707203058852982 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6707203058852982 - }, - "misleading_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6707203058852982 - } - } - }, - { - "id": "unseen01-de", - "expected": [ - "unseen01" - ], - "language": "de", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6558122105336331 - }, - "original": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.6588859671020654 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6558122105336331 - }, - "misleading_metadata": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen04", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.6588859671020654 - } - } - }, - { - "id": "unseen02-en", - "expected": [ - "unseen02" - ], - "language": "en", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6690030556732439 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen02", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7723038021644638 - }, - "missing_metadata": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6690030556732439 - }, - "misleading_metadata": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7723038021644638 - } - } - }, - { - "id": "unseen02-de", - "expected": [ - "unseen02" - ], - "language": "de", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.667712688085295 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen02", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7157722798891778 - }, - "missing_metadata": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.667712688085295 - }, - "misleading_metadata": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7157722798891778 - } - } - }, - { - "id": "unseen03-en", - "expected": [ - "unseen03" - ], - "language": "en", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6901545281005906 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6901545281005906 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6901545281005906 - }, - "misleading_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6901545281005906 - } - } - }, - { - "id": "unseen03-de", - "expected": [ - "unseen03" - ], - "language": "de", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6759388311775514 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6759388311775514 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6759388311775514 - }, - "misleading_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen03", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6759388311775514 - } - } - }, - { - "id": "unseen04-en", - "expected": [ - "unseen04" - ], - "language": "en", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7212670979690754 - }, - "original": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7212670979690754 - }, - "missing_metadata": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7212670979690754 - }, - "misleading_metadata": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7212670979690754 - } - } - }, - { - "id": "unseen04-de", - "expected": [ - "unseen04" - ], - "language": "de", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7094035244502205 - }, - "original": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7094035244502205 - }, - "missing_metadata": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7094035244502205 - }, - "misleading_metadata": { - "first_relevant_rank": 3, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7094035244502205 - } - } - }, - { - "id": "unseen05-en", - "expected": [ - "unseen05" - ], - "language": "en", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6986278621913614 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7804412810940734 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6986278621913614 - }, - "misleading_metadata": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen04", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7804412810940734 - } - } - }, - { - "id": "unseen05-de", - "expected": [ - "unseen05" - ], - "language": "de", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7104005169326468 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7544482777868109 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7104005169326468 - }, - "misleading_metadata": { - "first_relevant_rank": 2, - "returned_count": 10, - "first_file": "unseen04", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7544482777868109 - } - } - }, - { - "id": "unseen06-en", - "expected": [ - "unseen06" - ], - "language": "en", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7048521660816882 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen06", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7621619204850741 - }, - "missing_metadata": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7048521660816882 - }, - "misleading_metadata": { - "first_relevant_rank": 5, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7621619204850741 - } - } - }, - { - "id": "unseen06-de", - "expected": [ - "unseen06" - ], - "language": "de", - "category": "instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7064829076815999 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen06", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7392143518999531 - }, - "missing_metadata": { - "first_relevant_rank": 4, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7064829076815999 - }, - "misleading_metadata": { - "first_relevant_rank": 5, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7392143518999531 - } - } - }, - { - "id": "sax-broad-en", - "expected": [ - "unseen05", - "unseen06" - ], - "language": "en", - "category": "related-positive", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7506144568545938 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen06", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7558704108318935 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7506144568545938 - }, - "misleading_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.7558704108318935 - } - } - }, - { - "id": "sax-broad-de", - "expected": [ - "unseen05", - "unseen06" - ], - "language": "de", - "category": "related-positive", - "lanes": { - "audio_only": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7499595252586609 - }, - "original": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen06", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.762964312138735 - }, - "missing_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.7499595252586609 - }, - "misleading_metadata": { - "first_relevant_rank": 1, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.762964312138735 - } - } - }, - { - "id": "absent-bagpipe-en", - "expected": [], - "language": "en", - "category": "absent-instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6899061391978294 - }, - "original": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6899061391978294 - }, - "missing_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6899061391978294 - }, - "misleading_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6899061391978294 - } - } - }, - { - "id": "absent-bagpipe-de", - "expected": [], - "language": "de", - "category": "absent-instrument", - "lanes": { - "audio_only": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6517667406795858 - }, - "original": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6517667406795858 - }, - "missing_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6517667406795858 - }, - "misleading_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.6517667406795858 - } - } - }, - { - "id": "absent-artist-en", - "expected": [], - "language": "en", - "category": "absent-identity", - "lanes": { - "audio_only": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "sound02", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.5977197385044184 - }, - "original": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen05", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.5983290055413841 - }, - "missing_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "sound02", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.5977197385044184 - }, - "misleading_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen04", - "first_source_kind": "metadata", - "first_source_field": "title", - "first_score": 0.5983290055413841 - } - } - }, - { - "id": "absent-content-en", - "expected": [], - "language": "en", - "category": "absent-content", - "lanes": { - "audio_only": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.5870996954366186 - }, - "original": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.5870996954366186 - }, - "missing_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.5870996954366186 - }, - "misleading_metadata": { - "first_relevant_rank": null, - "returned_count": 10, - "first_file": "unseen01", - "first_source_kind": "audio", - "first_source_field": null, - "first_score": 0.5870996954366186 - } - } - } - ], - "environment": { - "os": "Windows11 Pro10.0.26300", - "logical_cpus": 8, - "visible_memory_kib": 16708012, - "processor": "AMD Ryzen7 7800X3D", - "shared_vm": true, - "existing_app_query_worker": "Warmed CPU session remained idle; no root model requests during reserved slot" - }, - "production_policy_changed": false, - "deployment_conclusion": "Original metadata improves aggregate retrieval but introduces a previously-correct German guitar failure; misleading metadata reduces top1 and no-answer remains unsolved. Unconditional max policy not validated for deployment. No post-score policy tuning." -} +{ + "source_commit": "cbf07b0", + "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", + "prepared_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78", + "raw_report_sha256": "69bcef5ffa58c0f23e5f34641256255bf15f54c836f75fb1aa878b4bd3df1309", + "raw_report_bytes": 267667, + "worker_sha256": "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73", + "worker": { + "code_query_supported": true, + "context_tokens": 2048, + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "role": "indexing", + "runtime": "b11461-no-logits-v1", + "threads": 1 + }, + "encoding_ms": 88089.5138, + "media_inference_calls": 10, + "metadata_inference_calls": 16, + "metrics": { + "audio_only": { + "positive_queries": 14, + "top1": 8, + "recall_at_5": 14, + "mrr": 0.7261904761904762, + "no_answer_queries": 4, + "no_answer_returned_any": 4 + }, + "original": { + "positive_queries": 14, + "top1": 11, + "recall_at_5": 14, + "mrr": 0.869047619047619, + "no_answer_queries": 4, + "no_answer_returned_any": 4 + }, + "missing_metadata": { + "positive_queries": 14, + "top1": 8, + "recall_at_5": 14, + "mrr": 0.7261904761904762, + "no_answer_queries": 4, + "no_answer_returned_any": 4 + }, + "misleading_metadata": { + "positive_queries": 14, + "top1": 5, + "recall_at_5": 14, + "mrr": 0.5702380952380952, + "no_answer_queries": 4, + "no_answer_returned_any": 4 + } + }, + "changes_vs_audio": { + "original": { + "top1_gains_vs_audio": [ + "unseen02-en", + "unseen02-de", + "unseen06-en", + "unseen06-de" + ], + "top1_losses_vs_audio": [ + "unseen01-de" + ] + }, + "missing_metadata": { + "top1_gains_vs_audio": [], + "top1_losses_vs_audio": [] + }, + "misleading_metadata": { + "top1_gains_vs_audio": [], + "top1_losses_vs_audio": [ + "unseen01-de", + "unseen05-en", + "unseen05-de" + ] + } + }, + "rows": [ + { + "id": "unseen01-en", + "expected": [ + "unseen01" + ], + "language": "en", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6707203058852982 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6707203058852982 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6707203058852982 + }, + "misleading_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6707203058852982 + } + } + }, + { + "id": "unseen01-de", + "expected": [ + "unseen01" + ], + "language": "de", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6558122105336331 + }, + "original": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.6588859671020654 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6558122105336331 + }, + "misleading_metadata": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen04", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.6588859671020654 + } + } + }, + { + "id": "unseen02-en", + "expected": [ + "unseen02" + ], + "language": "en", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6690030556732439 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen02", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7723038021644638 + }, + "missing_metadata": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6690030556732439 + }, + "misleading_metadata": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7723038021644638 + } + } + }, + { + "id": "unseen02-de", + "expected": [ + "unseen02" + ], + "language": "de", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.667712688085295 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen02", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7157722798891778 + }, + "missing_metadata": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.667712688085295 + }, + "misleading_metadata": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7157722798891778 + } + } + }, + { + "id": "unseen03-en", + "expected": [ + "unseen03" + ], + "language": "en", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6901545281005906 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6901545281005906 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6901545281005906 + }, + "misleading_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6901545281005906 + } + } + }, + { + "id": "unseen03-de", + "expected": [ + "unseen03" + ], + "language": "de", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6759388311775514 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6759388311775514 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6759388311775514 + }, + "misleading_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen03", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6759388311775514 + } + } + }, + { + "id": "unseen04-en", + "expected": [ + "unseen04" + ], + "language": "en", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7212670979690754 + }, + "original": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7212670979690754 + }, + "missing_metadata": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7212670979690754 + }, + "misleading_metadata": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7212670979690754 + } + } + }, + { + "id": "unseen04-de", + "expected": [ + "unseen04" + ], + "language": "de", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7094035244502205 + }, + "original": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7094035244502205 + }, + "missing_metadata": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7094035244502205 + }, + "misleading_metadata": { + "first_relevant_rank": 3, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7094035244502205 + } + } + }, + { + "id": "unseen05-en", + "expected": [ + "unseen05" + ], + "language": "en", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6986278621913614 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7804412810940734 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6986278621913614 + }, + "misleading_metadata": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen04", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7804412810940734 + } + } + }, + { + "id": "unseen05-de", + "expected": [ + "unseen05" + ], + "language": "de", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7104005169326468 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7544482777868109 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7104005169326468 + }, + "misleading_metadata": { + "first_relevant_rank": 2, + "returned_count": 10, + "first_file": "unseen04", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7544482777868109 + } + } + }, + { + "id": "unseen06-en", + "expected": [ + "unseen06" + ], + "language": "en", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7048521660816882 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen06", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7621619204850741 + }, + "missing_metadata": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7048521660816882 + }, + "misleading_metadata": { + "first_relevant_rank": 5, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7621619204850741 + } + } + }, + { + "id": "unseen06-de", + "expected": [ + "unseen06" + ], + "language": "de", + "category": "instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7064829076815999 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen06", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7392143518999531 + }, + "missing_metadata": { + "first_relevant_rank": 4, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7064829076815999 + }, + "misleading_metadata": { + "first_relevant_rank": 5, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7392143518999531 + } + } + }, + { + "id": "sax-broad-en", + "expected": [ + "unseen05", + "unseen06" + ], + "language": "en", + "category": "related-positive", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7506144568545938 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen06", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7558704108318935 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7506144568545938 + }, + "misleading_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.7558704108318935 + } + } + }, + { + "id": "sax-broad-de", + "expected": [ + "unseen05", + "unseen06" + ], + "language": "de", + "category": "related-positive", + "lanes": { + "audio_only": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7499595252586609 + }, + "original": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen06", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.762964312138735 + }, + "missing_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.7499595252586609 + }, + "misleading_metadata": { + "first_relevant_rank": 1, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.762964312138735 + } + } + }, + { + "id": "absent-bagpipe-en", + "expected": [], + "language": "en", + "category": "absent-instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6899061391978294 + }, + "original": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6899061391978294 + }, + "missing_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6899061391978294 + }, + "misleading_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6899061391978294 + } + } + }, + { + "id": "absent-bagpipe-de", + "expected": [], + "language": "de", + "category": "absent-instrument", + "lanes": { + "audio_only": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6517667406795858 + }, + "original": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6517667406795858 + }, + "missing_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6517667406795858 + }, + "misleading_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.6517667406795858 + } + } + }, + { + "id": "absent-artist-en", + "expected": [], + "language": "en", + "category": "absent-identity", + "lanes": { + "audio_only": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "sound02", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.5977197385044184 + }, + "original": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen05", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.5983290055413841 + }, + "missing_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "sound02", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.5977197385044184 + }, + "misleading_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen04", + "first_source_kind": "metadata", + "first_source_field": "title", + "first_score": 0.5983290055413841 + } + } + }, + { + "id": "absent-content-en", + "expected": [], + "language": "en", + "category": "absent-content", + "lanes": { + "audio_only": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.5870996954366186 + }, + "original": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.5870996954366186 + }, + "missing_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.5870996954366186 + }, + "misleading_metadata": { + "first_relevant_rank": null, + "returned_count": 10, + "first_file": "unseen01", + "first_source_kind": "audio", + "first_source_field": null, + "first_score": 0.5870996954366186 + } + } + } + ], + "environment": { + "os": "Windows11 Pro10.0.26300", + "logical_cpus": 8, + "visible_memory_kib": 16708012, + "processor": "AMD Ryzen7 7800X3D", + "shared_vm": true, + "existing_app_query_worker": "Warmed CPU session remained idle; no root model requests during reserved slot" + }, + "production_policy_changed": false, + "deployment_conclusion": "Original metadata improves aggregate retrieval but introduces a previously-correct German guitar failure; misleading metadata reduces top1 and no-answer remains unsolved. Unconditional max policy not validated for deployment. No post-score policy tuning." +} diff --git a/fixtures/music-validation-v1/fixture.json b/fixtures/music-validation-v1/fixture.json index 6750ce8..240353a 100644 --- a/fixtures/music-validation-v1/fixture.json +++ b/fixtures/music-validation-v1/fixture.json @@ -1,268 +1,268 @@ -{ - "version": 1, - "frozen_before_inference": true, - "assets": [ - { - "id": "unseen01", - "filename": "AcousticGuitarSample.ogg", - "url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", - "source_page": "https://commons.wikimedia.org/wiki/File:AcousticGuitarSample.ogg", - "bytes": 889069, - "sha1": "d622d15b17b5c101631dd7707a57cf24b9c474da", - "license": "Public domain", - "license_url": null, - "credit_html": "RyGuy", - "publisher_description_html": "Sample of the sounds which can be made by an acoustic guitar.", - "clip_start_ms": 0, - "clip_duration_ms": 30000, - "label_basis": "Publisher description only; human listening and second annotator unexecuted." - }, - { - "id": "unseen02", - "filename": "Cello strings.ogg", - "url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg", - "source_page": "https://commons.wikimedia.org/wiki/File:Cello_strings.ogg", - "bytes": 296474, - "sha1": "8c965bc8679ac6b84b6abeaa086311892d590cae", - "license": "CC BY-SA 3.0", - "license_url": "http://creativecommons.org/licenses/by-sa/3.0/", - "credit_html": null, - "publisher_description_html": null, - "clip_start_ms": 0, - "clip_duration_ms": 30000, - "label_basis": "Publisher description only; human listening and second annotator unexecuted." - }, - { - "id": "unseen03", - "filename": "Drum - Cadence A.ogg", - "url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg", - "source_page": "https://commons.wikimedia.org/wiki/File:Drum_-_Cadence_A.ogg", - "bytes": 1574547, - "sha1": "19d4fbf201e4990d9b2be9c454b209f36af86700", - "license": "Public domain", - "license_url": null, - "credit_html": "Unknown authorUnknown author", - "publisher_description_html": "Drum Cadence A performed by the United States Navy Band", - "clip_start_ms": 0, - "clip_duration_ms": 30000, - "label_basis": "Publisher description only; human listening and second annotator unexecuted." - }, - { - "id": "unseen04", - "filename": "Bucket mute trumpet demo.ogg", - "url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg", - "source_page": "https://commons.wikimedia.org/wiki/File:Bucket_mute_trumpet_demo.ogg", - "bytes": 228972, - "sha1": "37ebe58d5d3eb6051b066ed31bcc66240518b86c", - "license": "CC BY-SA 4.0", - "license_url": "https://creativecommons.org/licenses/by-sa/4.0", - "credit_html": "Ovinus Real", - "publisher_description_html": "I play the concert notes F3, Bb3, Eb4, Ab4, Db5, Gb5, B5 in succession on a Bb trumpet with bucket mute inserted. I then play a short jazz-ish excerpt (that I had previously improvised) to demonstrate the character of the tone. The sound was amplified but not processed in any other way.", - "clip_start_ms": 0, - "clip_duration_ms": 30000, - "label_basis": "Publisher description only; human listening and second annotator unexecuted." - }, - { - "id": "unseen05", - "filename": "Afteryouvegone-first8bars.ogg", - "url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg", - "source_page": "https://commons.wikimedia.org/wiki/File:Afteryouvegone-first8bars.ogg", - "bytes": 300001, - "sha1": "92eee317927ff5cdc5614fe848350cfde2419df3", - "license": "CC BY-SA 3.0", - "license_url": "https://creativecommons.org/licenses/by-sa/3.0", - "credit_html": "Julien Grandgagnage", - "publisher_description_html": "After You've Gone - first 8 bars on tenor saxophone", - "clip_start_ms": 0, - "clip_duration_ms": 30000, - "label_basis": "Publisher description only; human listening and second annotator unexecuted." - }, - { - "id": "unseen06", - "filename": "Aintshesweet8bars.ogg", - "url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg", - "source_page": "https://commons.wikimedia.org/wiki/File:Aintshesweet8bars.ogg", - "bytes": 250846, - "sha1": "3fb787158e3250e8a30f99d82fd9ee0f4743511c", - "license": "CC BY-SA 3.0", - "license_url": "https://creativecommons.org/licenses/by-sa/3.0", - "credit_html": "Julien Grandgagnage", - "publisher_description_html": "Ain't She Sweet, first 8 bars on soprano saxophone", - "clip_start_ms": 0, - "clip_duration_ms": 30000, - "label_basis": "Publisher description only; human listening and second annotator unexecuted." - } - ], - "queries": [ - { - "id": "unseen01-en", - "text": "a recording of acoustic guitar", - "language": "en", - "expected": [ - "unseen01" - ], - "category": "instrument" - }, - { - "id": "unseen01-de", - "text": "eine Aufnahme mit akustische Gitarre", - "language": "de", - "expected": [ - "unseen01" - ], - "category": "instrument" - }, - { - "id": "unseen02-en", - "text": "a recording of cello strings", - "language": "en", - "expected": [ - "unseen02" - ], - "category": "instrument" - }, - { - "id": "unseen02-de", - "text": "eine Aufnahme mit Cellosaiten", - "language": "de", - "expected": [ - "unseen02" - ], - "category": "instrument" - }, - { - "id": "unseen03-en", - "text": "a recording of military drum cadence", - "language": "en", - "expected": [ - "unseen03" - ], - "category": "instrument" - }, - { - "id": "unseen03-de", - "text": "eine Aufnahme mit militärische Trommelkadenz", - "language": "de", - "expected": [ - "unseen03" - ], - "category": "instrument" - }, - { - "id": "unseen04-en", - "text": "a recording of trumpet with a bucket mute", - "language": "en", - "expected": [ - "unseen04" - ], - "category": "instrument" - }, - { - "id": "unseen04-de", - "text": "eine Aufnahme mit Trompete mit Bucket-Dämpfer", - "language": "de", - "expected": [ - "unseen04" - ], - "category": "instrument" - }, - { - "id": "unseen05-en", - "text": "a recording of tenor saxophone melody", - "language": "en", - "expected": [ - "unseen05" - ], - "category": "instrument" - }, - { - "id": "unseen05-de", - "text": "eine Aufnahme mit Tenorsaxophon-Melodie", - "language": "de", - "expected": [ - "unseen05" - ], - "category": "instrument" - }, - { - "id": "unseen06-en", - "text": "a recording of soprano saxophone melody", - "language": "en", - "expected": [ - "unseen06" - ], - "category": "instrument" - }, - { - "id": "unseen06-de", - "text": "eine Aufnahme mit Sopransaxophon-Melodie", - "language": "de", - "expected": [ - "unseen06" - ], - "category": "instrument" - }, - { - "id": "sax-broad-en", - "text": "a saxophone playing a melody", - "language": "en", - "expected": [ - "unseen05", - "unseen06" - ], - "category": "related-positive" - }, - { - "id": "sax-broad-de", - "text": "eine Saxophonmelodie", - "language": "de", - "expected": [ - "unseen05", - "unseen06" - ], - "category": "related-positive" - }, - { - "id": "absent-bagpipe-en", - "text": "a bagpipe performance", - "language": "en", - "expected": [], - "category": "absent-instrument" - }, - { - "id": "absent-bagpipe-de", - "text": "eine Dudelsackaufnahme", - "language": "de", - "expected": [], - "category": "absent-instrument" - }, - { - "id": "absent-artist-en", - "text": "Adele singing Hello", - "language": "en", - "expected": [], - "category": "absent-identity" - }, - { - "id": "absent-content-en", - "text": "a person explaining quantum entanglement", - "language": "en", - "expected": [], - "category": "absent-content" - } - ], - "variants": [ - "original", - "missing_metadata", - "misleading_metadata" - ], - "misleading_policy": "Whitelisted actual tags cyclically transferred from the next asset; no invented descriptive tags. All clips encode first <=30seconds as lossless FLAC mono16kHz, metadata stripped or replaced. Verify exact decoded PCM SHA256 equality across variants before reusing audio vectors. Report controlled corruption separately.", - "candidate_policy": "New six sources plus optional predeclared legacy image/audio/video distractors, fixed before inference; never treat related plausible music as absent. Broad sax labels credit both tenor and soprano. Source title is not instrument evidence for the raw-audio lane.", - "metadata_policy": "Whitelisted artist/title/album/genre/composer fields separately, total2048 UTF8bytes, neutral File metadata title, file maximum over real metadata and audio, noweights/nocutoff.", - "legacy_distractors": [ - "image01", - "image02", - "sound01", - "sound02" - ] -} +{ + "version": 1, + "frozen_before_inference": true, + "assets": [ + { + "id": "unseen01", + "filename": "AcousticGuitarSample.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:AcousticGuitarSample.ogg", + "bytes": 889069, + "sha1": "d622d15b17b5c101631dd7707a57cf24b9c474da", + "license": "Public domain", + "license_url": null, + "credit_html": "RyGuy", + "publisher_description_html": "Sample of the sounds which can be made by an acoustic guitar.", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen02", + "filename": "Cello strings.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Cello_strings.ogg", + "bytes": 296474, + "sha1": "8c965bc8679ac6b84b6abeaa086311892d590cae", + "license": "CC BY-SA 3.0", + "license_url": "http://creativecommons.org/licenses/by-sa/3.0/", + "credit_html": null, + "publisher_description_html": null, + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen03", + "filename": "Drum - Cadence A.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Drum_-_Cadence_A.ogg", + "bytes": 1574547, + "sha1": "19d4fbf201e4990d9b2be9c454b209f36af86700", + "license": "Public domain", + "license_url": null, + "credit_html": "Unknown authorUnknown author", + "publisher_description_html": "Drum Cadence A performed by the United States Navy Band", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen04", + "filename": "Bucket mute trumpet demo.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Bucket_mute_trumpet_demo.ogg", + "bytes": 228972, + "sha1": "37ebe58d5d3eb6051b066ed31bcc66240518b86c", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "credit_html": "Ovinus Real", + "publisher_description_html": "I play the concert notes F3, Bb3, Eb4, Ab4, Db5, Gb5, B5 in succession on a Bb trumpet with bucket mute inserted. I then play a short jazz-ish excerpt (that I had previously improvised) to demonstrate the character of the tone. The sound was amplified but not processed in any other way.", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen05", + "filename": "Afteryouvegone-first8bars.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Afteryouvegone-first8bars.ogg", + "bytes": 300001, + "sha1": "92eee317927ff5cdc5614fe848350cfde2419df3", + "license": "CC BY-SA 3.0", + "license_url": "https://creativecommons.org/licenses/by-sa/3.0", + "credit_html": "Julien Grandgagnage", + "publisher_description_html": "After You've Gone - first 8 bars on tenor saxophone", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + }, + { + "id": "unseen06", + "filename": "Aintshesweet8bars.ogg", + "url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Aintshesweet8bars.ogg", + "bytes": 250846, + "sha1": "3fb787158e3250e8a30f99d82fd9ee0f4743511c", + "license": "CC BY-SA 3.0", + "license_url": "https://creativecommons.org/licenses/by-sa/3.0", + "credit_html": "Julien Grandgagnage", + "publisher_description_html": "Ain't She Sweet, first 8 bars on soprano saxophone", + "clip_start_ms": 0, + "clip_duration_ms": 30000, + "label_basis": "Publisher description only; human listening and second annotator unexecuted." + } + ], + "queries": [ + { + "id": "unseen01-en", + "text": "a recording of acoustic guitar", + "language": "en", + "expected": [ + "unseen01" + ], + "category": "instrument" + }, + { + "id": "unseen01-de", + "text": "eine Aufnahme mit akustische Gitarre", + "language": "de", + "expected": [ + "unseen01" + ], + "category": "instrument" + }, + { + "id": "unseen02-en", + "text": "a recording of cello strings", + "language": "en", + "expected": [ + "unseen02" + ], + "category": "instrument" + }, + { + "id": "unseen02-de", + "text": "eine Aufnahme mit Cellosaiten", + "language": "de", + "expected": [ + "unseen02" + ], + "category": "instrument" + }, + { + "id": "unseen03-en", + "text": "a recording of military drum cadence", + "language": "en", + "expected": [ + "unseen03" + ], + "category": "instrument" + }, + { + "id": "unseen03-de", + "text": "eine Aufnahme mit militärische Trommelkadenz", + "language": "de", + "expected": [ + "unseen03" + ], + "category": "instrument" + }, + { + "id": "unseen04-en", + "text": "a recording of trumpet with a bucket mute", + "language": "en", + "expected": [ + "unseen04" + ], + "category": "instrument" + }, + { + "id": "unseen04-de", + "text": "eine Aufnahme mit Trompete mit Bucket-Dämpfer", + "language": "de", + "expected": [ + "unseen04" + ], + "category": "instrument" + }, + { + "id": "unseen05-en", + "text": "a recording of tenor saxophone melody", + "language": "en", + "expected": [ + "unseen05" + ], + "category": "instrument" + }, + { + "id": "unseen05-de", + "text": "eine Aufnahme mit Tenorsaxophon-Melodie", + "language": "de", + "expected": [ + "unseen05" + ], + "category": "instrument" + }, + { + "id": "unseen06-en", + "text": "a recording of soprano saxophone melody", + "language": "en", + "expected": [ + "unseen06" + ], + "category": "instrument" + }, + { + "id": "unseen06-de", + "text": "eine Aufnahme mit Sopransaxophon-Melodie", + "language": "de", + "expected": [ + "unseen06" + ], + "category": "instrument" + }, + { + "id": "sax-broad-en", + "text": "a saxophone playing a melody", + "language": "en", + "expected": [ + "unseen05", + "unseen06" + ], + "category": "related-positive" + }, + { + "id": "sax-broad-de", + "text": "eine Saxophonmelodie", + "language": "de", + "expected": [ + "unseen05", + "unseen06" + ], + "category": "related-positive" + }, + { + "id": "absent-bagpipe-en", + "text": "a bagpipe performance", + "language": "en", + "expected": [], + "category": "absent-instrument" + }, + { + "id": "absent-bagpipe-de", + "text": "eine Dudelsackaufnahme", + "language": "de", + "expected": [], + "category": "absent-instrument" + }, + { + "id": "absent-artist-en", + "text": "Adele singing Hello", + "language": "en", + "expected": [], + "category": "absent-identity" + }, + { + "id": "absent-content-en", + "text": "a person explaining quantum entanglement", + "language": "en", + "expected": [], + "category": "absent-content" + } + ], + "variants": [ + "original", + "missing_metadata", + "misleading_metadata" + ], + "misleading_policy": "Whitelisted actual tags cyclically transferred from the next asset; no invented descriptive tags. All clips encode first <=30seconds as lossless FLAC mono16kHz, metadata stripped or replaced. Verify exact decoded PCM SHA256 equality across variants before reusing audio vectors. Report controlled corruption separately.", + "candidate_policy": "New six sources plus optional predeclared legacy image/audio/video distractors, fixed before inference; never treat related plausible music as absent. Broad sax labels credit both tenor and soprano. Source title is not instrument evidence for the raw-audio lane.", + "metadata_policy": "Whitelisted artist/title/album/genre/composer fields separately, total2048 UTF8bytes, neutral File metadata title, file maximum over real metadata and audio, noweights/nocutoff.", + "legacy_distractors": [ + "image01", + "image02", + "sound01", + "sound02" + ] +} diff --git a/fixtures/music-validation-v1/inputs.lock.json b/fixtures/music-validation-v1/inputs.lock.json index 2c09aea..1793b51 100644 --- a/fixtures/music-validation-v1/inputs.lock.json +++ b/fixtures/music-validation-v1/inputs.lock.json @@ -1,303 +1,303 @@ -{ - "frozen_before_inference": true, - "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", - "prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78", - "sources": [ - { - "id": "unseen01", - "sha256": "1c604cce5299b2f9579538810deb328d9c47607f098f5885ff1daa7cf34b637a", - "bytes": 889069, - "source_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", - "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", - "actual_tags": { - "artist": "Ryan" - } - }, - { - "id": "unseen02", - "sha256": "f62b0dabedd570f3abd9fc9e9a3a468375bb77ddd7f1ad02c203c8d5394bf219", - "bytes": 296474, - "source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg", - "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg?download=1", - "actual_tags": { - "title": "cello strings", - "date": "06/10/2005" - } - }, - { - "id": "unseen03", - "sha256": "5083f07730c6ffb1cd2c3b5cc1a43714a0cfda04fe913935e83d1048466562e6", - "bytes": 1574547, - "source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg", - "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg?download=1", - "actual_tags": {} - }, - { - "id": "unseen04", - "sha256": "7fc61102e3050c5fe01000b1b02f839e725b305333505075916faeea78844caa", - "bytes": 228972, - "source_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg", - "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg?download=1", - "actual_tags": { - "artist": "Ovinus Real", - "comments": "First played a series of ascending fourths from concert F3 to B5, then replayed the improvised excerpt from the harmon mute (without stem) recording. No processing or noise removal done.", - "date": "2020", - "title": "Bucket mute" - } - }, - { - "id": "unseen05", - "sha256": "0570623af1490ed81b86c5a95f5b95eb61b9d941bb0116e4271e61a07d2dc5ba", - "bytes": 300001, - "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg", - "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg?download=1", - "actual_tags": { - "title": "After you've gone - first 8 bars on tenor sax", - "date": "2013", - "artist": "Beachcomber" - } - }, - { - "id": "unseen06", - "sha256": "8f2035ab8924ee9f0d212de5853cf22eab7457520c1d774413a78601ed1b8bd6", - "bytes": 250846, - "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg", - "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg?download=1", - "actual_tags": { - "title": "Ain't She Sweet, first 8 bars on soprano sax", - "date": "2013", - "artist": "Beachcomber" - } - } - ], - "containers": [ - { - "id": "unseen01", - "variant": "original", - "kind": "audio", - "sha256": "24b14ccc428e96312a96a556e5250b3d4c781bb42e4549f13b43ddc2fb31c087", - "actual_tags": { - "artist": "Ryan", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" - }, - { - "id": "unseen01", - "variant": "missing_metadata", - "kind": "audio", - "sha256": "665ddf557d94a65ff98f1d35563657405846577351de08f00b5a104bb1d6f992", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" - }, - { - "id": "unseen01", - "variant": "misleading_metadata", - "kind": "audio", - "sha256": "bac63053fc9182e17269ec1015ebf2bc74b803abfb4f52c25a0009f145c7065a", - "actual_tags": { - "title": "cello strings", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" - }, - { - "id": "unseen02", - "variant": "original", - "kind": "audio", - "sha256": "fa934449cb45743ffa420c412e3b7bc388ae846e76b29573922ab9506496bbfb", - "actual_tags": { - "title": "cello strings", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" - }, - { - "id": "unseen02", - "variant": "missing_metadata", - "kind": "audio", - "sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" - }, - { - "id": "unseen02", - "variant": "misleading_metadata", - "kind": "audio", - "sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" - }, - { - "id": "unseen03", - "variant": "original", - "kind": "audio", - "sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" - }, - { - "id": "unseen03", - "variant": "missing_metadata", - "kind": "audio", - "sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" - }, - { - "id": "unseen03", - "variant": "misleading_metadata", - "kind": "audio", - "sha256": "5e108abbbc44fe068b07a5b124894c4738204f511a308c4385960019f5c453a1", - "actual_tags": { - "artist": "Ovinus Real", - "title": "Bucket mute", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" - }, - { - "id": "unseen04", - "variant": "original", - "kind": "audio", - "sha256": "fc498bc0e47fc97b8c085cd00266d3e2dbebecfc3021e443b7b19e43dd025dde", - "actual_tags": { - "artist": "Ovinus Real", - "title": "Bucket mute", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" - }, - { - "id": "unseen04", - "variant": "missing_metadata", - "kind": "audio", - "sha256": "09b2df6db7953ed67e5ae5059b3adacbec7e5c13a3be6aa32237b19e106e9fac", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" - }, - { - "id": "unseen04", - "variant": "misleading_metadata", - "kind": "audio", - "sha256": "fdaae3a8636394100b268a4a2ea8b42565be87fcb8fbdaefc9bee29a42d77b8a", - "actual_tags": { - "artist": "Beachcomber", - "title": "After you've gone - first 8 bars on tenor sax", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" - }, - { - "id": "unseen05", - "variant": "original", - "kind": "audio", - "sha256": "caa19f57058fb7a1d7ac715c087a6c8430f9054b7a3c503ab7b191794f805652", - "actual_tags": { - "artist": "Beachcomber", - "title": "After you've gone - first 8 bars on tenor sax", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" - }, - { - "id": "unseen05", - "variant": "missing_metadata", - "kind": "audio", - "sha256": "1e46c0ec9c8831fa041d27c46cba924803a29fe5304bf5776d598db297b0b24f", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" - }, - { - "id": "unseen05", - "variant": "misleading_metadata", - "kind": "audio", - "sha256": "56217ac0a4a85921e939c7a84fbe864e1d8c9e1d7b48b6ba54d64187370423aa", - "actual_tags": { - "artist": "Beachcomber", - "title": "Ain't She Sweet, first 8 bars on soprano sax", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" - }, - { - "id": "unseen06", - "variant": "original", - "kind": "audio", - "sha256": "27b4a28c800ac7b949af7f92d1e483bc4089c98a45a381b52c816254ef3810fc", - "actual_tags": { - "artist": "Beachcomber", - "title": "Ain't She Sweet, first 8 bars on soprano sax", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" - }, - { - "id": "unseen06", - "variant": "missing_metadata", - "kind": "audio", - "sha256": "603b253da7ee235284ab8bb9cdac177001093122ce385032b6a2a00d4ffcb1e8", - "actual_tags": { - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" - }, - { - "id": "unseen06", - "variant": "misleading_metadata", - "kind": "audio", - "sha256": "f8b0e7f4261f69fa326370433dfd6519c09e9a63f12fcaf9dbf47fe8cf481fe2", - "actual_tags": { - "artist": "Ryan", - "encoder": "Lavf63.1.102" - }, - "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" - }, - { - "id": "image01", - "variant": "fixed", - "kind": "image", - "sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed", - "actual_tags": {} - }, - { - "id": "image02", - "variant": "fixed", - "kind": "image", - "sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1", - "actual_tags": {} - }, - { - "id": "sound01", - "variant": "fixed", - "kind": "audio", - "sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451", - "actual_tags": {} - }, - { - "id": "sound02", - "variant": "fixed", - "kind": "audio", - "sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e", - "actual_tags": { - "encoder": "LAME3.98r" - } - } - ], - "ffmpeg_sha256": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46", - "ffprobe_sha256": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236" -} +{ + "frozen_before_inference": true, + "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", + "prepared_plan_sha256": "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78", + "sources": [ + { + "id": "unseen01", + "sha256": "1c604cce5299b2f9579538810deb328d9c47607f098f5885ff1daa7cf34b637a", + "bytes": 889069, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/6/67/AcousticGuitarSample.ogg", + "actual_tags": { + "artist": "Ryan" + } + }, + { + "id": "unseen02", + "sha256": "f62b0dabedd570f3abd9fc9e9a3a468375bb77ddd7f1ad02c203c8d5394bf219", + "bytes": 296474, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Cello_strings.ogg?download=1", + "actual_tags": { + "title": "cello strings", + "date": "06/10/2005" + } + }, + { + "id": "unseen03", + "sha256": "5083f07730c6ffb1cd2c3b5cc1a43714a0cfda04fe913935e83d1048466562e6", + "bytes": 1574547, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/e/e1/Drum_-_Cadence_A.ogg?download=1", + "actual_tags": {} + }, + { + "id": "unseen04", + "sha256": "7fc61102e3050c5fe01000b1b02f839e725b305333505075916faeea78844caa", + "bytes": 228972, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Bucket_mute_trumpet_demo.ogg?download=1", + "actual_tags": { + "artist": "Ovinus Real", + "comments": "First played a series of ascending fourths from concert F3 to B5, then replayed the improvised excerpt from the harmon mute (without stem) recording. No processing or noise removal done.", + "date": "2020", + "title": "Bucket mute" + } + }, + { + "id": "unseen05", + "sha256": "0570623af1490ed81b86c5a95f5b95eb61b9d941bb0116e4271e61a07d2dc5ba", + "bytes": 300001, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Afteryouvegone-first8bars.ogg?download=1", + "actual_tags": { + "title": "After you've gone - first 8 bars on tenor sax", + "date": "2013", + "artist": "Beachcomber" + } + }, + { + "id": "unseen06", + "sha256": "8f2035ab8924ee9f0d212de5853cf22eab7457520c1d774413a78601ed1b8bd6", + "bytes": 250846, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg", + "actual_requested_url": "https://upload.wikimedia.org/wikipedia/commons/5/59/Aintshesweet8bars.ogg?download=1", + "actual_tags": { + "title": "Ain't She Sweet, first 8 bars on soprano sax", + "date": "2013", + "artist": "Beachcomber" + } + } + ], + "containers": [ + { + "id": "unseen01", + "variant": "original", + "kind": "audio", + "sha256": "24b14ccc428e96312a96a556e5250b3d4c781bb42e4549f13b43ddc2fb31c087", + "actual_tags": { + "artist": "Ryan", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" + }, + { + "id": "unseen01", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "665ddf557d94a65ff98f1d35563657405846577351de08f00b5a104bb1d6f992", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" + }, + { + "id": "unseen01", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "bac63053fc9182e17269ec1015ebf2bc74b803abfb4f52c25a0009f145c7065a", + "actual_tags": { + "title": "cello strings", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "e6c153958edc7651b8f805c59377b8ef996efe568690d9489709230373bf52e7" + }, + { + "id": "unseen02", + "variant": "original", + "kind": "audio", + "sha256": "fa934449cb45743ffa420c412e3b7bc388ae846e76b29573922ab9506496bbfb", + "actual_tags": { + "title": "cello strings", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" + }, + { + "id": "unseen02", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" + }, + { + "id": "unseen02", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "0cef37d960d5fa28a9381d0d53b919862a33052d976cf076837e5bc5c029c072", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "a49388dc0c5a8b41ffb400995d176a13ecb9c9652e508755e08f2ab659c11a7a" + }, + { + "id": "unseen03", + "variant": "original", + "kind": "audio", + "sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" + }, + { + "id": "unseen03", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "9e77ccb17648a3f4c18ec317c3a1cee0b7dbdd1b0a9ff892106762ccc80e10c7", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" + }, + { + "id": "unseen03", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "5e108abbbc44fe068b07a5b124894c4738204f511a308c4385960019f5c453a1", + "actual_tags": { + "artist": "Ovinus Real", + "title": "Bucket mute", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "9b67be21743e6ed33d3f04d2d7c91bacd90a9554132ad7027b4349d7d50d4932" + }, + { + "id": "unseen04", + "variant": "original", + "kind": "audio", + "sha256": "fc498bc0e47fc97b8c085cd00266d3e2dbebecfc3021e443b7b19e43dd025dde", + "actual_tags": { + "artist": "Ovinus Real", + "title": "Bucket mute", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" + }, + { + "id": "unseen04", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "09b2df6db7953ed67e5ae5059b3adacbec7e5c13a3be6aa32237b19e106e9fac", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" + }, + { + "id": "unseen04", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "fdaae3a8636394100b268a4a2ea8b42565be87fcb8fbdaefc9bee29a42d77b8a", + "actual_tags": { + "artist": "Beachcomber", + "title": "After you've gone - first 8 bars on tenor sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "d31fc644485355da47fe82a3151398c4400096b7d48bcb32edc776cb9a237c46" + }, + { + "id": "unseen05", + "variant": "original", + "kind": "audio", + "sha256": "caa19f57058fb7a1d7ac715c087a6c8430f9054b7a3c503ab7b191794f805652", + "actual_tags": { + "artist": "Beachcomber", + "title": "After you've gone - first 8 bars on tenor sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" + }, + { + "id": "unseen05", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "1e46c0ec9c8831fa041d27c46cba924803a29fe5304bf5776d598db297b0b24f", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" + }, + { + "id": "unseen05", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "56217ac0a4a85921e939c7a84fbe864e1d8c9e1d7b48b6ba54d64187370423aa", + "actual_tags": { + "artist": "Beachcomber", + "title": "Ain't She Sweet, first 8 bars on soprano sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "76c9e866a5fe0b27f8015b5f1be11757039db2ec403b04cbb6c7ca7b57a977d3" + }, + { + "id": "unseen06", + "variant": "original", + "kind": "audio", + "sha256": "27b4a28c800ac7b949af7f92d1e483bc4089c98a45a381b52c816254ef3810fc", + "actual_tags": { + "artist": "Beachcomber", + "title": "Ain't She Sweet, first 8 bars on soprano sax", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" + }, + { + "id": "unseen06", + "variant": "missing_metadata", + "kind": "audio", + "sha256": "603b253da7ee235284ab8bb9cdac177001093122ce385032b6a2a00d4ffcb1e8", + "actual_tags": { + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" + }, + { + "id": "unseen06", + "variant": "misleading_metadata", + "kind": "audio", + "sha256": "f8b0e7f4261f69fa326370433dfd6519c09e9a63f12fcaf9dbf47fe8cf481fe2", + "actual_tags": { + "artist": "Ryan", + "encoder": "Lavf63.1.102" + }, + "pcm_sha256": "d58d0208860dcbad037c5f76edb82d2b0263a278773a31876e4f8c4361d9e261" + }, + { + "id": "image01", + "variant": "fixed", + "kind": "image", + "sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed", + "actual_tags": {} + }, + { + "id": "image02", + "variant": "fixed", + "kind": "image", + "sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1", + "actual_tags": {} + }, + { + "id": "sound01", + "variant": "fixed", + "kind": "audio", + "sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451", + "actual_tags": {} + }, + { + "id": "sound02", + "variant": "fixed", + "kind": "audio", + "sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e", + "actual_tags": { + "encoder": "LAME3.98r" + } + } + ], + "ffmpeg_sha256": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46", + "ffprobe_sha256": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236" +} diff --git a/fixtures/music-validation-v1/labels.lock.json b/fixtures/music-validation-v1/labels.lock.json index b224296..8a591c5 100644 --- a/fixtures/music-validation-v1/labels.lock.json +++ b/fixtures/music-validation-v1/labels.lock.json @@ -1,6 +1,6 @@ -{ - "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", - "frozen_before_inference": true, - "query_count": 18, - "asset_count": 6 -} +{ + "fixture_sha256": "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577", + "frozen_before_inference": true, + "query_count": 18, + "asset_count": 6 +} diff --git a/tools/rerank-eval/compatibility-plan.json b/tools/rerank-eval/compatibility-plan.json index 867568a..9325eab 100644 --- a/tools/rerank-eval/compatibility-plan.json +++ b/tools/rerank-eval/compatibility-plan.json @@ -1,38 +1,38 @@ -{ - "version": 1, - "status": "pre-inference frozen compatibility/cost contract, not relevance calibration", - "cpu_provider": "CPUExecutionProvider", - "intra_op_threads": 1, - "inter_op_threads": 1, - "parallel_execution": false, - "max_pairs": 5, - "max_pair_tokens": 256, - "max_query_tokens": 64, - "warm_iterations": 20, - "parity_absolute_tolerance": 0.001, - "parity_relative_tolerance": 0.001, - "added_warm_p95_target_ms": 200, - "no_suppression": true, - "pairs": [ - [ - "How do we cancel the supplier partnership?", - "Either party may terminate the supplier contract by giving sixty days written notice." - ], - [ - "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", - "Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen." - ], - [ - "How do we cancel the supplier partnership?", - "Tomatoes require sunshine and consistent watering." - ], - [ - "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", - "Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung." - ], - [ - "Where does the code implement retry backoff?", - "fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }" - ] - ] -} +{ + "version": 1, + "status": "pre-inference frozen compatibility/cost contract, not relevance calibration", + "cpu_provider": "CPUExecutionProvider", + "intra_op_threads": 1, + "inter_op_threads": 1, + "parallel_execution": false, + "max_pairs": 5, + "max_pair_tokens": 256, + "max_query_tokens": 64, + "warm_iterations": 20, + "parity_absolute_tolerance": 0.001, + "parity_relative_tolerance": 0.001, + "added_warm_p95_target_ms": 200, + "no_suppression": true, + "pairs": [ + [ + "How do we cancel the supplier partnership?", + "Either party may terminate the supplier contract by giving sixty days written notice." + ], + [ + "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", + "Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen." + ], + [ + "How do we cancel the supplier partnership?", + "Tomatoes require sunshine and consistent watering." + ], + [ + "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", + "Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung." + ], + [ + "Where does the code implement retry backoff?", + "fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }" + ] + ] +} diff --git a/tools/rerank-eval/model-manifest.json b/tools/rerank-eval/model-manifest.json index 09c6261..04ea259 100644 --- a/tools/rerank-eval/model-manifest.json +++ b/tools/rerank-eval/model-manifest.json @@ -1,50 +1,50 @@ -{ - "model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", - "revision": "1427fd652930e4ba29e8149678df786c240d8825", - "purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval", - "license": "Apache-2.0 per pinned publisher card; text from Apache", - "files": [ - { - "path": "onnx/model_quint8_avx2.onnx", - "bytes": 118620016, - "sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx" - }, - { - "path": "tokenizer.json", - "bytes": 17082660, - "sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json" - }, - { - "path": "config.json", - "bytes": 891, - "sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json" - }, - { - "path": "tokenizer_config.json", - "bytes": 435, - "sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json" - }, - { - "path": "special_tokens_map.json", - "bytes": 239, - "sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json" - }, - { - "path": "README.md", - "bytes": 2278, - "sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md" - }, - { - "path": "LICENSE-APACHE-2.0.txt", - "bytes": 11358, - "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", - "url": "https://www.apache.org/licenses/LICENSE-2.0.txt" - } - ] -} +{ + "model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", + "revision": "1427fd652930e4ba29e8149678df786c240d8825", + "purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval", + "license": "Apache-2.0 per pinned publisher card; text from Apache", + "files": [ + { + "path": "onnx/model_quint8_avx2.onnx", + "bytes": 118620016, + "sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx" + }, + { + "path": "tokenizer.json", + "bytes": 17082660, + "sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json" + }, + { + "path": "config.json", + "bytes": 891, + "sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json" + }, + { + "path": "tokenizer_config.json", + "bytes": 435, + "sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json" + }, + { + "path": "special_tokens_map.json", + "bytes": 239, + "sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json" + }, + { + "path": "README.md", + "bytes": 2278, + "sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md" + }, + { + "path": "LICENSE-APACHE-2.0.txt", + "bytes": 11358, + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + "url": "https://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] +} diff --git a/tools/rerank-eval/reference-manifest.json b/tools/rerank-eval/reference-manifest.json index 78e618b..d0528a2 100644 --- a/tools/rerank-eval/reference-manifest.json +++ b/tools/rerank-eval/reference-manifest.json @@ -1,56 +1,56 @@ -{ - "model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", - "revision": "1427fd652930e4ba29e8149678df786c240d8825", - "purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval", - "license": "Apache-2.0 per pinned publisher card; text from Apache", - "files": [ - { - "path": "onnx/model_quint8_avx2.onnx", - "bytes": 118620016, - "sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx" - }, - { - "path": "tokenizer.json", - "bytes": 17082660, - "sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json" - }, - { - "path": "config.json", - "bytes": 891, - "sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json" - }, - { - "path": "tokenizer_config.json", - "bytes": 435, - "sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json" - }, - { - "path": "special_tokens_map.json", - "bytes": 239, - "sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json" - }, - { - "path": "README.md", - "bytes": 2278, - "sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md" - }, - { - "path": "LICENSE-APACHE-2.0.txt", - "bytes": 11358, - "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", - "url": "https://www.apache.org/licenses/LICENSE-2.0.txt" - }, - { - "path": "onnx/model.onnx", - "bytes": 470883696, - "sha256": "3e9a03ed1e966f7c5288dd4230e3d6a9bf5e3a170a06f1f4241c5bca12c6487c", - "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model.onnx" - } - ] -} +{ + "model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", + "revision": "1427fd652930e4ba29e8149678df786c240d8825", + "purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval", + "license": "Apache-2.0 per pinned publisher card; text from Apache", + "files": [ + { + "path": "onnx/model_quint8_avx2.onnx", + "bytes": 118620016, + "sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx" + }, + { + "path": "tokenizer.json", + "bytes": 17082660, + "sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json" + }, + { + "path": "config.json", + "bytes": 891, + "sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json" + }, + { + "path": "tokenizer_config.json", + "bytes": 435, + "sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json" + }, + { + "path": "special_tokens_map.json", + "bytes": 239, + "sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json" + }, + { + "path": "README.md", + "bytes": 2278, + "sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md" + }, + { + "path": "LICENSE-APACHE-2.0.txt", + "bytes": 11358, + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + "url": "https://www.apache.org/licenses/LICENSE-2.0.txt" + }, + { + "path": "onnx/model.onnx", + "bytes": 470883696, + "sha256": "3e9a03ed1e966f7c5288dd4230e3d6a9bf5e3a170a06f1f4241c5bca12c6487c", + "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model.onnx" + } + ] +} diff --git a/tools/rerank-eval/reference-plan.json b/tools/rerank-eval/reference-plan.json index 1b00582..7cb13f2 100644 --- a/tools/rerank-eval/reference-plan.json +++ b/tools/rerank-eval/reference-plan.json @@ -1,44 +1,44 @@ -{ - "version": 2, - "status": "pre-inference F32 reference parity and 256-token cost contract; original INT8 failure unchanged", - "cpu_provider": "CPUExecutionProvider", - "intra_op_threads": 1, - "inter_op_threads": 1, - "parallel_execution": false, - "max_pairs": 6, - "max_pair_tokens": 256, - "max_query_tokens": 64, - "warm_iterations": 20, - "parity_absolute_tolerance": 0.001, - "parity_relative_tolerance": 0.001, - "added_warm_p95_target_ms": 200, - "no_suppression": true, - "pairs": [ - [ - "How do we cancel the supplier partnership?", - "Either party may terminate the supplier contract by giving sixty days written notice." - ], - [ - "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", - "Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen." - ], - [ - "How do we cancel the supplier partnership?", - "Tomatoes require sunshine and consistent watering." - ], - [ - "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", - "Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung." - ], - [ - "Where does the code implement retry backoff?", - "fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }" - ], - [ - "supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice ", - "Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. " - ] - ], - "reference_model": "onnx/model.onnx", - "comparison_model": "onnx/model_quint8_avx2.onnx" -} +{ + "version": 2, + "status": "pre-inference F32 reference parity and 256-token cost contract; original INT8 failure unchanged", + "cpu_provider": "CPUExecutionProvider", + "intra_op_threads": 1, + "inter_op_threads": 1, + "parallel_execution": false, + "max_pairs": 6, + "max_pair_tokens": 256, + "max_query_tokens": 64, + "warm_iterations": 20, + "parity_absolute_tolerance": 0.001, + "parity_relative_tolerance": 0.001, + "added_warm_p95_target_ms": 200, + "no_suppression": true, + "pairs": [ + [ + "How do we cancel the supplier partnership?", + "Either party may terminate the supplier contract by giving sixty days written notice." + ], + [ + "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", + "Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen." + ], + [ + "How do we cancel the supplier partnership?", + "Tomatoes require sunshine and consistent watering." + ], + [ + "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", + "Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung." + ], + [ + "Where does the code implement retry backoff?", + "fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }" + ], + [ + "supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice ", + "Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. " + ] + ], + "reference_model": "onnx/model.onnx", + "comparison_model": "onnx/model_quint8_avx2.onnx" +}