407 lines
14 KiB
JSON
407 lines
14 KiB
JSON
|
{
|
||
|
|
"source_commit": "2f9190c",
|
||
|
|
"evaluator_commit": "48f11de",
|
||
|
|
"service_source_commit": "dab7034",
|
||
|
|
"fixture_sha256": "e3a850d7a2385e250408f57fe51f4620ce80a20f8625ced42205f02cc46d517d",
|
||
|
|
"prepared_sha256": "085f34f0af863656a28e7a905746973a54d1123bb07053541e09c8af0a58bb3f",
|
||
|
|
"binary_sha256": "7aa75b7765b7567a3f6761c6b72449092d5adab7c3a59aaa8e55ed622340e910",
|
||
|
|
"speech_worker_sha256": "299c1b784eac46b31d228453a399e1378f327fc9ed5e34ab950c804da33e433a",
|
||
|
|
"speech_inventory_sha256": "4904995685a5e55e0a790d9cc409dc8458e39c30cda4097c5d70bf75cb5c976c",
|
||
|
|
"google_worker_sha256": "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73",
|
||
|
|
"base_sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe",
|
||
|
|
"vad_sha256": "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",
|
||
|
|
"speech_policy": "live-window-vad-dtw-v1;silero-v6.2.0;base;30s;vad-reset;threshold0.5;minspeech250ms;minsilence100ms;maxspeechFLT_MAX;pad30ms;vad-overlap0.1s;source-sample-intersections;beam5;bestof5;temp0;tempinc0.2;entropy2.4;logprob-1;nospeech0.6;suppressblank;no-suppress-nst;auto-language;no-context;dtw-base;no-flash;contiguous-intervals;first-last-token-points;exact-utf8",
|
||
|
|
"artifact_hashes": {
|
||
|
|
"preflight.json": {
|
||
|
|
"bytes": 2075,
|
||
|
|
"sha256": "63a0a615d37766b852c34473712156e950aeeb5a472a709bff6be00b7b26cb7a"
|
||
|
|
},
|
||
|
|
"report.json": {
|
||
|
|
"bytes": 174868,
|
||
|
|
"sha256": "79ecb75d31c0193642c81862fd5476ad47adca518d4959e436bdd472cc281798"
|
||
|
|
},
|
||
|
|
"scored.json": {
|
||
|
|
"bytes": 5608,
|
||
|
|
"sha256": "21d9bc84410cc63b96e2b575d99cd3c09834dfd9d2db2d0f356159df12e048e9"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"results": {
|
||
|
|
"comparison_eligible": true,
|
||
|
|
"strict_point_score_eligible": false,
|
||
|
|
"positive_queries": 6,
|
||
|
|
"top1": 6,
|
||
|
|
"recall_at_5": 6,
|
||
|
|
"mrr": 1.0,
|
||
|
|
"negative_queries": 4,
|
||
|
|
"negative_returned_any": 4,
|
||
|
|
"native_passage_counts": {
|
||
|
|
"music-control": 0,
|
||
|
|
"silence-control": 0,
|
||
|
|
"speech-de": 3,
|
||
|
|
"speech-en": 4
|
||
|
|
},
|
||
|
|
"negative_gate_acceptance": true,
|
||
|
|
"reference_concept_substring_coverage": {
|
||
|
|
"speech-en": {
|
||
|
|
"free": true,
|
||
|
|
"equal": true,
|
||
|
|
"dignity": true,
|
||
|
|
"rights": true,
|
||
|
|
"reason": true,
|
||
|
|
"conscience": true,
|
||
|
|
"brotherhood": true
|
||
|
|
},
|
||
|
|
"speech-de": {
|
||
|
|
"frei": true,
|
||
|
|
"gleich": true,
|
||
|
|
"Würde": true,
|
||
|
|
"Rechten": true,
|
||
|
|
"Vernunft": true,
|
||
|
|
"Gewissen": true,
|
||
|
|
"Brüderlichkeit": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"rows": [
|
||
|
|
{
|
||
|
|
"id": "equal-en",
|
||
|
|
"language": "en",
|
||
|
|
"positive": true,
|
||
|
|
"first_relevant_rank": 1,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-de",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 7430
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "equal-de",
|
||
|
|
"language": "de",
|
||
|
|
"positive": true,
|
||
|
|
"first_relevant_rank": 1,
|
||
|
|
"returned_count": 2,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-de",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 7430
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "reason-en",
|
||
|
|
"language": "en",
|
||
|
|
"positive": true,
|
||
|
|
"first_relevant_rank": 1,
|
||
|
|
"returned_count": 2,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-en",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " they are endowed with reason and conscience.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 14730
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "reason-de",
|
||
|
|
"language": "de",
|
||
|
|
"positive": true,
|
||
|
|
"first_relevant_rank": 1,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-de",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geisterbrüderlichkeit begegnen.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 11100
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "solidarity-en",
|
||
|
|
"language": "en",
|
||
|
|
"positive": true,
|
||
|
|
"first_relevant_rank": 1,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-en",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " and should act towards one another in a spirit of brotherhood.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 18430
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "solidarity-de",
|
||
|
|
"language": "de",
|
||
|
|
"positive": true,
|
||
|
|
"first_relevant_rank": 1,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-de",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geisterbrüderlichkeit begegnen.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 11100
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "absent-mars-en",
|
||
|
|
"language": "en",
|
||
|
|
"positive": false,
|
||
|
|
"first_relevant_rank": null,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-en",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " they are endowed with reason and conscience.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 14730
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "absent-mars-de",
|
||
|
|
"language": "de",
|
||
|
|
"positive": false,
|
||
|
|
"first_relevant_rank": null,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-en",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " they are endowed with reason and conscience.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 14730
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "absent-salary-en",
|
||
|
|
"language": "en",
|
||
|
|
"positive": false,
|
||
|
|
"first_relevant_rank": null,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-en",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " they are endowed with reason and conscience.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 14730
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "absent-salary-de",
|
||
|
|
"language": "de",
|
||
|
|
"positive": false,
|
||
|
|
"first_relevant_rank": null,
|
||
|
|
"returned_count": 4,
|
||
|
|
"first_hit": {
|
||
|
|
"asset": "speech-de",
|
||
|
|
"match_kind": "semantic_speech",
|
||
|
|
"snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.",
|
||
|
|
"source": {
|
||
|
|
"kind": "timestamp_ms",
|
||
|
|
"number": 7430
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"cases": {
|
||
|
|
"core": {
|
||
|
|
"expected_native_calls": 4,
|
||
|
|
"actual_native_calls": 4,
|
||
|
|
"index_report": {
|
||
|
|
"audio_inference_calls": 4,
|
||
|
|
"chunks": 11,
|
||
|
|
"document_inference_calls": 0,
|
||
|
|
"encoded_sources": 8,
|
||
|
|
"failures": [],
|
||
|
|
"generation": 1,
|
||
|
|
"image_inference_calls": 0,
|
||
|
|
"media_failures": 0,
|
||
|
|
"media_files": 4,
|
||
|
|
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1:speech-v1-422af6d2bbbaaa3e943700c10035116aefcee24f35cd909bf33646bc41e00980",
|
||
|
|
"passages": 0,
|
||
|
|
"resumed": false,
|
||
|
|
"reused_chunks": 0,
|
||
|
|
"reused_sources": 0,
|
||
|
|
"speech_chunks": 7,
|
||
|
|
"speech_failures": [],
|
||
|
|
"speech_metrics": {
|
||
|
|
"admission_us": 0,
|
||
|
|
"asr_us": 61574700,
|
||
|
|
"fallback_us": 0,
|
||
|
|
"google_embedding_calls": 7,
|
||
|
|
"load_us": 209078,
|
||
|
|
"transport_us": 212365,
|
||
|
|
"vad_us": 216972
|
||
|
|
},
|
||
|
|
"speech_requested": true,
|
||
|
|
"speech_reused_sources": 0,
|
||
|
|
"speech_windows": 4
|
||
|
|
},
|
||
|
|
"indexing_ms": 127611.1518,
|
||
|
|
"speech_load_verify_ms": 3330.588,
|
||
|
|
"google_index_load_verify_ms": 7431.4775,
|
||
|
|
"google_query_load_verify_ms": 7615.0823,
|
||
|
|
"query_warmup_ms": 128.039,
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"id": "speech-en",
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"asset": "speech-en",
|
||
|
|
"duration_ms": 22512,
|
||
|
|
"pcm_sha256": "56948226ac5a0ad4b70523a856dad0851929c7067957f339e9a5b30fc9b369ce",
|
||
|
|
"samples": 360192,
|
||
|
|
"start_ms": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "speech-de",
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"asset": "speech-de",
|
||
|
|
"duration_ms": 16219,
|
||
|
|
"pcm_sha256": "136a1c03298a55f0ea7869e84a4684d7bd9e9fca4477b69ba860e7e300712ca2",
|
||
|
|
"samples": 259504,
|
||
|
|
"start_ms": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "music-control",
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"asset": "music-control",
|
||
|
|
"duration_ms": 30000,
|
||
|
|
"pcm_sha256": "a1fd7233bb4f9088688cfb36e449fb68b8bd5cdc70f2d193371ed9587b905193",
|
||
|
|
"samples": 480000,
|
||
|
|
"start_ms": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "silence-control",
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"asset": "silence-control",
|
||
|
|
"duration_ms": 30000,
|
||
|
|
"pcm_sha256": "124617c1f65e92d3bc895fbd869e4bb16a30754b198f59e6e973949b9aaa1b01",
|
||
|
|
"samples": 480000,
|
||
|
|
"start_ms": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"seams": {
|
||
|
|
"expected_native_calls": 4,
|
||
|
|
"actual_native_calls": 4,
|
||
|
|
"index_report": {
|
||
|
|
"audio_inference_calls": 4,
|
||
|
|
"chunks": 24,
|
||
|
|
"document_inference_calls": 0,
|
||
|
|
"encoded_sources": 4,
|
||
|
|
"failures": [],
|
||
|
|
"generation": 1,
|
||
|
|
"image_inference_calls": 0,
|
||
|
|
"media_failures": 0,
|
||
|
|
"media_files": 2,
|
||
|
|
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1:speech-v1-422af6d2bbbaaa3e943700c10035116aefcee24f35cd909bf33646bc41e00980",
|
||
|
|
"passages": 0,
|
||
|
|
"resumed": false,
|
||
|
|
"reused_chunks": 0,
|
||
|
|
"reused_sources": 0,
|
||
|
|
"speech_chunks": 20,
|
||
|
|
"speech_failures": [],
|
||
|
|
"speech_metrics": {
|
||
|
|
"admission_us": 0,
|
||
|
|
"asr_us": 121231168,
|
||
|
|
"fallback_us": 0,
|
||
|
|
"google_embedding_calls": 20,
|
||
|
|
"load_us": 213591,
|
||
|
|
"transport_us": 109395,
|
||
|
|
"vad_us": 187061
|
||
|
|
},
|
||
|
|
"speech_requested": true,
|
||
|
|
"speech_reused_sources": 0,
|
||
|
|
"speech_windows": 4
|
||
|
|
},
|
||
|
|
"indexing_ms": 163367.8024,
|
||
|
|
"speech_load_verify_ms": 3011.0661,
|
||
|
|
"google_index_load_verify_ms": 7365.343099999999,
|
||
|
|
"google_query_load_verify_ms": 7039.3844,
|
||
|
|
"query_warmup_ms": 115.2243,
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"id": "seam-en",
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"asset": "seam-en",
|
||
|
|
"duration_ms": 30000,
|
||
|
|
"pcm_sha256": "7b9e755453e90f67f4ee9f1a6c4aaa5048225faeb266e9e187c890baac6a7138",
|
||
|
|
"samples": 480000,
|
||
|
|
"start_ms": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"asset": "seam-en",
|
||
|
|
"duration_ms": 12024,
|
||
|
|
"pcm_sha256": "40b1bac0855bee11643dcce5d2e38801c5d3533993bd035f6a3c70ef219f180f",
|
||
|
|
"samples": 192384,
|
||
|
|
"start_ms": 28000
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "seam-de",
|
||
|
|
"windows": [
|
||
|
|
{
|
||
|
|
"asset": "seam-de",
|
||
|
|
"duration_ms": 30000,
|
||
|
|
"pcm_sha256": "9b77a526b62d1f473a014626eb312f995b3dd4b788c96bc31981edd243c50eea",
|
||
|
|
"samples": 480000,
|
||
|
|
"start_ms": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"asset": "seam-de",
|
||
|
|
"duration_ms": 10657,
|
||
|
|
"pcm_sha256": "5d5c5988ab40601479b72498e7a63dbfedfe0567572597da52811b63c0a5d23b",
|
||
|
|
"samples": 170512,
|
||
|
|
"start_ms": 28000
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"limitations": [
|
||
|
|
"InternalCPUv3 runtime; normalAuto/GPU behavior not evaluated",
|
||
|
|
"Native normalAuto compilation jobs1 overlapped sharedVM timings; no controlled hardware throughput claim",
|
||
|
|
"Translated content correlated; concept substrings are not WER or exact transcript fidelity",
|
||
|
|
"German Geisterbrüderlichkeit and Artikel1s recognized incorrectly; seam partial/duplicate text retained",
|
||
|
|
"No timed publisher phrase groundtruth: strictpoint scoring unavailable",
|
||
|
|
"All4 noanswer queries return files, no abstention improvement",
|
||
|
|
"Accepted publicSpeechTranscription values recorded, no native wire/stderr capture"
|
||
|
|
],
|
||
|
|
"production_policy_changed": false
|
||
|
|
}
|