2026-10-13 19:01:08 +02:00
{
"source_commit" : "cbf07b0" ,
"fixture_sha256" : "f9d7785ff937020b802e6c56252e64bbc3cd22e9a09709ddce9b82f9adc1b577" ,
"prepared_sha256" : "3e874108914e8f0334da8f12b704261e83755bd25dce6f93e22e8ce22b8dee78" ,
"raw_report_sha256" : "69bcef5ffa58c0f23e5f34641256255bf15f54c836f75fb1aa878b4bd3df1309" ,
"raw_report_bytes" : 267667 ,
"worker_sha256" : "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73" ,
"worker" : {
"code_query_supported" : true ,
"context_tokens" : 2048 ,
"device" : "cpu" ,
"dimensions" : 768 ,
"media_supported" : true ,
"model" : "google/embeddinggemma-2" ,
"ready" : true ,
"role" : "indexing" ,
"runtime" : "b11461-no-logits-v1" ,
"threads" : 1
},
"encoding_ms" : 88089.5138 ,
"media_inference_calls" : 10 ,
"metadata_inference_calls" : 16 ,
"metrics" : {
"audio_only" : {
"positive_queries" : 14 ,
"top1" : 8 ,
"recall_at_5" : 14 ,
"mrr" : 0.7261904761904762 ,
"no_answer_queries" : 4 ,
"no_answer_returned_any" : 4
},
"original" : {
"positive_queries" : 14 ,
"top1" : 11 ,
"recall_at_5" : 14 ,
"mrr" : 0.869047619047619 ,
"no_answer_queries" : 4 ,
"no_answer_returned_any" : 4
},
"missing_metadata" : {
"positive_queries" : 14 ,
"top1" : 8 ,
"recall_at_5" : 14 ,
"mrr" : 0.7261904761904762 ,
"no_answer_queries" : 4 ,
"no_answer_returned_any" : 4
},
"misleading_metadata" : {
"positive_queries" : 14 ,
"top1" : 5 ,
"recall_at_5" : 14 ,
"mrr" : 0.5702380952380952 ,
"no_answer_queries" : 4 ,
"no_answer_returned_any" : 4
}
},
"changes_vs_audio" : {
"original" : {
"top1_gains_vs_audio" : [
"unseen02-en" ,
"unseen02-de" ,
"unseen06-en" ,
"unseen06-de"
],
"top1_losses_vs_audio" : [
"unseen01-de"
]
},
"missing_metadata" : {
"top1_gains_vs_audio" : [],
"top1_losses_vs_audio" : []
},
"misleading_metadata" : {
"top1_gains_vs_audio" : [],
"top1_losses_vs_audio" : [
"unseen01-de" ,
"unseen05-en" ,
"unseen05-de"
]
}
},
"rows" : [
{
"id" : "unseen01-en" ,
"expected" : [
"unseen01"
],
"language" : "en" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6707203058852982
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6707203058852982
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6707203058852982
},
"misleading_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6707203058852982
}
}
},
{
"id" : "unseen01-de" ,
"expected" : [
"unseen01"
],
"language" : "de" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6558122105336331
},
"original" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.6588859671020654
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6558122105336331
},
"misleading_metadata" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen04" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.6588859671020654
}
}
},
{
"id" : "unseen02-en" ,
"expected" : [
"unseen02"
],
"language" : "en" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6690030556732439
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen02" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7723038021644638
},
"missing_metadata" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6690030556732439
},
"misleading_metadata" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7723038021644638
}
}
},
{
"id" : "unseen02-de" ,
"expected" : [
"unseen02"
],
"language" : "de" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.667712688085295
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen02" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7157722798891778
},
"missing_metadata" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.667712688085295
},
"misleading_metadata" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7157722798891778
}
}
},
{
"id" : "unseen03-en" ,
"expected" : [
"unseen03"
],
"language" : "en" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6901545281005906
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6901545281005906
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6901545281005906
},
"misleading_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6901545281005906
}
}
},
{
"id" : "unseen03-de" ,
"expected" : [
"unseen03"
],
"language" : "de" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6759388311775514
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6759388311775514
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6759388311775514
},
"misleading_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen03" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6759388311775514
}
}
},
{
"id" : "unseen04-en" ,
"expected" : [
"unseen04"
],
"language" : "en" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7212670979690754
},
"original" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7212670979690754
},
"missing_metadata" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7212670979690754
},
"misleading_metadata" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7212670979690754
}
}
},
{
"id" : "unseen04-de" ,
"expected" : [
"unseen04"
],
"language" : "de" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7094035244502205
},
"original" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7094035244502205
},
"missing_metadata" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7094035244502205
},
"misleading_metadata" : {
"first_relevant_rank" : 3 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7094035244502205
}
}
},
{
"id" : "unseen05-en" ,
"expected" : [
"unseen05"
],
"language" : "en" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6986278621913614
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7804412810940734
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6986278621913614
},
"misleading_metadata" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen04" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7804412810940734
}
}
},
{
"id" : "unseen05-de" ,
"expected" : [
"unseen05"
],
"language" : "de" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7104005169326468
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7544482777868109
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7104005169326468
},
"misleading_metadata" : {
"first_relevant_rank" : 2 ,
"returned_count" : 10 ,
"first_file" : "unseen04" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7544482777868109
}
}
},
{
"id" : "unseen06-en" ,
"expected" : [
"unseen06"
],
"language" : "en" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7048521660816882
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen06" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7621619204850741
},
"missing_metadata" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7048521660816882
},
"misleading_metadata" : {
"first_relevant_rank" : 5 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7621619204850741
}
}
},
{
"id" : "unseen06-de" ,
"expected" : [
"unseen06"
],
"language" : "de" ,
"category" : "instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7064829076815999
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen06" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7392143518999531
},
"missing_metadata" : {
"first_relevant_rank" : 4 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7064829076815999
},
"misleading_metadata" : {
"first_relevant_rank" : 5 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7392143518999531
}
}
},
{
"id" : "sax-broad-en" ,
"expected" : [
"unseen05" ,
"unseen06"
],
"language" : "en" ,
"category" : "related-positive" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7506144568545938
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen06" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7558704108318935
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7506144568545938
},
"misleading_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.7558704108318935
}
}
},
{
"id" : "sax-broad-de" ,
"expected" : [
"unseen05" ,
"unseen06"
],
"language" : "de" ,
"category" : "related-positive" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7499595252586609
},
"original" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen06" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.762964312138735
},
"missing_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.7499595252586609
},
"misleading_metadata" : {
"first_relevant_rank" : 1 ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.762964312138735
}
}
},
{
"id" : "absent-bagpipe-en" ,
"expected" : [],
"language" : "en" ,
"category" : "absent-instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6899061391978294
},
"original" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6899061391978294
},
"missing_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6899061391978294
},
"misleading_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6899061391978294
}
}
},
{
"id" : "absent-bagpipe-de" ,
"expected" : [],
"language" : "de" ,
"category" : "absent-instrument" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6517667406795858
},
"original" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6517667406795858
},
"missing_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6517667406795858
},
"misleading_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.6517667406795858
}
}
},
{
"id" : "absent-artist-en" ,
"expected" : [],
"language" : "en" ,
"category" : "absent-identity" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "sound02" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.5977197385044184
},
"original" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen05" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.5983290055413841
},
"missing_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "sound02" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.5977197385044184
},
"misleading_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen04" ,
"first_source_kind" : "metadata" ,
"first_source_field" : "title" ,
"first_score" : 0.5983290055413841
}
}
},
{
"id" : "absent-content-en" ,
"expected" : [],
"language" : "en" ,
"category" : "absent-content" ,
"lanes" : {
"audio_only" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.5870996954366186
},
"original" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.5870996954366186
},
"missing_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.5870996954366186
},
"misleading_metadata" : {
"first_relevant_rank" : null ,
"returned_count" : 10 ,
"first_file" : "unseen01" ,
"first_source_kind" : "audio" ,
"first_source_field" : null ,
"first_score" : 0.5870996954366186
}
}
}
],
"environment" : {
"os" : "Windows11 Pro10.0.26300" ,
"logical_cpus" : 8 ,
"visible_memory_kib" : 16708012 ,
"processor" : "AMD Ryzen7 7800X3D" ,
"shared_vm" : true ,
"existing_app_query_worker" : "Warmed CPU session remained idle; no root model requests during reserved slot"
},
"production_policy_changed" : false ,
"deployment_conclusion" : "Original metadata improves aggregate retrieval but introduces a previously-correct German guitar failure; misleading metadata reduces top1 and no-answer remains unsolved. Unconditional max policy not validated for deployment. No post-score policy tuning."
}
