karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Freeze normal-runtime speech timing regression without timestamp selection

Browse Source
This commit is contained in:
karl committed 2026-10-13 20:46:31 +02:00
1 parent e68f4fcadc
commit 2c3c91179b
7 files changed
+819 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/speech-eval/native-validation/evaluation-v1-summary.json
+407
View File
@@ -0,0 +1,407 @@
{
"source_commit": "2f9190c",
"evaluator_commit": "48f11de",
"service_source_commit": "dab7034",
"fixture_sha256": "e3a850d7a2385e250408f57fe51f4620ce80a20f8625ced42205f02cc46d517d",
"prepared_sha256": "085f34f0af863656a28e7a905746973a54d1123bb07053541e09c8af0a58bb3f",
"binary_sha256": "7aa75b7765b7567a3f6761c6b72449092d5adab7c3a59aaa8e55ed622340e910",
"speech_worker_sha256": "299c1b784eac46b31d228453a399e1378f327fc9ed5e34ab950c804da33e433a",
"speech_inventory_sha256": "4904995685a5e55e0a790d9cc409dc8458e39c30cda4097c5d70bf75cb5c976c",
"google_worker_sha256": "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73",
"base_sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe",
"vad_sha256": "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",
"speech_policy": "live-window-vad-dtw-v1;silero-v6.2.0;base;30s;vad-reset;threshold0.5;minspeech250ms;minsilence100ms;maxspeechFLT_MAX;pad30ms;vad-overlap0.1s;source-sample-intersections;beam5;bestof5;temp0;tempinc0.2;entropy2.4;logprob-1;nospeech0.6;suppressblank;no-suppress-nst;auto-language;no-context;dtw-base;no-flash;contiguous-intervals;first-last-token-points;exact-utf8",
"artifact_hashes": {
"preflight.json": {
"bytes": 2075,
"sha256": "63a0a615d37766b852c34473712156e950aeeb5a472a709bff6be00b7b26cb7a"
},
"report.json": {
"bytes": 174868,
"sha256": "79ecb75d31c0193642c81862fd5476ad47adca518d4959e436bdd472cc281798"
},
"scored.json": {
"bytes": 5608,
"sha256": "21d9bc84410cc63b96e2b575d99cd3c09834dfd9d2db2d0f356159df12e048e9"
}
},
"results": {
"comparison_eligible": true,
"strict_point_score_eligible": false,
"positive_queries": 6,
"top1": 6,
"recall_at_5": 6,
"mrr": 1.0,
"negative_queries": 4,
"negative_returned_any": 4,
"native_passage_counts": {
"music-control": 0,
"silence-control": 0,
"speech-de": 3,
"speech-en": 4
},
"negative_gate_acceptance": true,
"reference_concept_substring_coverage": {
"speech-en": {
"free": true,
"equal": true,
"dignity": true,
"rights": true,
"reason": true,
"conscience": true,
"brotherhood": true
},
"speech-de": {
"frei": true,
"gleich": true,
"Würde": true,
"Rechten": true,
"Vernunft": true,
"Gewissen": true,
"Brüderlichkeit": true
}
},
"rows": [
{
"id": "equal-en",
"language": "en",
"positive": true,
"first_relevant_rank": 1,
"returned_count": 4,
"first_hit": {
"asset": "speech-de",
"match_kind": "semantic_speech",
"snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.",
"source": {
"kind": "timestamp_ms",
"number": 7430
}
}
},
{
"id": "equal-de",
"language": "de",
"positive": true,
"first_relevant_rank": 1,
"returned_count": 2,
"first_hit": {
"asset": "speech-de",
"match_kind": "semantic_speech",
"snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.",
"source": {
"kind": "timestamp_ms",
"number": 7430
}
}
},
{
"id": "reason-en",
"language": "en",
"positive": true,
"first_relevant_rank": 1,
"returned_count": 2,
"first_hit": {
"asset": "speech-en",
"match_kind": "semantic_speech",
"snippet": " they are endowed with reason and conscience.",
"source": {
"kind": "timestamp_ms",
"number": 14730
}
}
},
{
"id": "reason-de",
"language": "de",
"positive": true,
"first_relevant_rank": 1,
"returned_count": 4,
"first_hit": {
"asset": "speech-de",
"match_kind": "semantic_speech",
"snippet": " Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geisterbrüderlichkeit begegnen.",
"source": {
"kind": "timestamp_ms",
"number": 11100
}
}
},
{
"id": "solidarity-en",
"language": "en",
"positive": true,
"first_relevant_rank": 1,
"returned_count": 4,
"first_hit": {
"asset": "speech-en",
"match_kind": "semantic_speech",
"snippet": " and should act towards one another in a spirit of brotherhood.",
"source": {
"kind": "timestamp_ms",
"number": 18430
}
}
},
{
"id": "solidarity-de",
"language": "de",
"positive": true,
"first_relevant_rank": 1,
"returned_count": 4,
"first_hit": {
"asset": "speech-de",
"match_kind": "semantic_speech",
"snippet": " Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geisterbrüderlichkeit begegnen.",
"source": {
"kind": "timestamp_ms",
"number": 11100
}
}
},
{
"id": "absent-mars-en",
"language": "en",
"positive": false,
"first_relevant_rank": null,
"returned_count": 4,
"first_hit": {
"asset": "speech-en",
"match_kind": "semantic_speech",
"snippet": " they are endowed with reason and conscience.",
"source": {
"kind": "timestamp_ms",
"number": 14730
}
}
},
{
"id": "absent-mars-de",
"language": "de",
"positive": false,
"first_relevant_rank": null,
"returned_count": 4,
"first_hit": {
"asset": "speech-en",
"match_kind": "semantic_speech",
"snippet": " they are endowed with reason and conscience.",
"source": {
"kind": "timestamp_ms",
"number": 14730
}
}
},
{
"id": "absent-salary-en",
"language": "en",
"positive": false,
"first_relevant_rank": null,
"returned_count": 4,
"first_hit": {
"asset": "speech-en",
"match_kind": "semantic_speech",
"snippet": " they are endowed with reason and conscience.",
"source": {
"kind": "timestamp_ms",
"number": 14730
}
}
},
{
"id": "absent-salary-de",
"language": "de",
"positive": false,
"first_relevant_rank": null,
"returned_count": 4,
"first_hit": {
"asset": "speech-de",
"match_kind": "semantic_speech",
"snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.",
"source": {
"kind": "timestamp_ms",
"number": 7430
}
}
}
]
},
"cases": {
"core": {
"expected_native_calls": 4,
"actual_native_calls": 4,
"index_report": {
"audio_inference_calls": 4,
"chunks": 11,
"document_inference_calls": 0,
"encoded_sources": 8,
"failures": [],
"generation": 1,
"image_inference_calls": 0,
"media_failures": 0,
"media_files": 4,
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1:speech-v1-422af6d2bbbaaa3e943700c10035116aefcee24f35cd909bf33646bc41e00980",
"passages": 0,
"resumed": false,
"reused_chunks": 0,
"reused_sources": 0,
"speech_chunks": 7,
"speech_failures": [],
"speech_metrics": {
"admission_us": 0,
"asr_us": 61574700,
"fallback_us": 0,
"google_embedding_calls": 7,
"load_us": 209078,
"transport_us": 212365,
"vad_us": 216972
},
"speech_requested": true,
"speech_reused_sources": 0,
"speech_windows": 4
},
"indexing_ms": 127611.1518,
"speech_load_verify_ms": 3330.588,
"google_index_load_verify_ms": 7431.4775,
"google_query_load_verify_ms": 7615.0823,
"query_warmup_ms": 128.039,
"windows": [
{
"id": "speech-en",
"windows": [
{
"asset": "speech-en",
"duration_ms": 22512,
"pcm_sha256": "56948226ac5a0ad4b70523a856dad0851929c7067957f339e9a5b30fc9b369ce",
"samples": 360192,
"start_ms": 0
}
]
},
{
"id": "speech-de",
"windows": [
{
"asset": "speech-de",
"duration_ms": 16219,
"pcm_sha256": "136a1c03298a55f0ea7869e84a4684d7bd9e9fca4477b69ba860e7e300712ca2",
"samples": 259504,
"start_ms": 0
}
]
},
{
"id": "music-control",
"windows": [
{
"asset": "music-control",
"duration_ms": 30000,
"pcm_sha256": "a1fd7233bb4f9088688cfb36e449fb68b8bd5cdc70f2d193371ed9587b905193",
"samples": 480000,
"start_ms": 0
}
]
},
{
"id": "silence-control",
"windows": [
{
"asset": "silence-control",
"duration_ms": 30000,
"pcm_sha256": "124617c1f65e92d3bc895fbd869e4bb16a30754b198f59e6e973949b9aaa1b01",
"samples": 480000,
"start_ms": 0
}
]
}
]
},
"seams": {
"expected_native_calls": 4,
"actual_native_calls": 4,
"index_report": {
"audio_inference_calls": 4,
"chunks": 24,
"document_inference_calls": 0,
"encoded_sources": 4,
"failures": [],
"generation": 1,
"image_inference_calls": 0,
"media_failures": 0,
"media_files": 2,
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1:speech-v1-422af6d2bbbaaa3e943700c10035116aefcee24f35cd909bf33646bc41e00980",
"passages": 0,
"resumed": false,
"reused_chunks": 0,
"reused_sources": 0,
"speech_chunks": 20,
"speech_failures": [],
"speech_metrics": {
"admission_us": 0,
"asr_us": 121231168,
"fallback_us": 0,
"google_embedding_calls": 20,
"load_us": 213591,
"transport_us": 109395,
"vad_us": 187061
},
"speech_requested": true,
"speech_reused_sources": 0,
"speech_windows": 4
},
"indexing_ms": 163367.8024,
"speech_load_verify_ms": 3011.0661,
"google_index_load_verify_ms": 7365.343099999999,
"google_query_load_verify_ms": 7039.3844,
"query_warmup_ms": 115.2243,
"windows": [
{
"id": "seam-en",
"windows": [
{
"asset": "seam-en",
"duration_ms": 30000,
"pcm_sha256": "7b9e755453e90f67f4ee9f1a6c4aaa5048225faeb266e9e187c890baac6a7138",
"samples": 480000,
"start_ms": 0
},
{
"asset": "seam-en",
"duration_ms": 12024,
"pcm_sha256": "40b1bac0855bee11643dcce5d2e38801c5d3533993bd035f6a3c70ef219f180f",
"samples": 192384,
"start_ms": 28000
}
]
},
{
"id": "seam-de",
"windows": [
{
"asset": "seam-de",
"duration_ms": 30000,
"pcm_sha256": "9b77a526b62d1f473a014626eb312f995b3dd4b788c96bc31981edd243c50eea",
"samples": 480000,
"start_ms": 0
},
{
"asset": "seam-de",
"duration_ms": 10657,
"pcm_sha256": "5d5c5988ab40601479b72498e7a63dbfedfe0567572597da52811b63c0a5d23b",
"samples": 170512,
"start_ms": 28000
}
]
}
]
}
},
"limitations": [
"InternalCPUv3 runtime; normalAuto/GPU behavior not evaluated",
"Native normalAuto compilation jobs1 overlapped sharedVM timings; no controlled hardware throughput claim",
"Translated content correlated; concept substrings are not WER or exact transcript fidelity",
"German Geisterbrüderlichkeit and Artikel1s recognized incorrectly; seam partial/duplicate text retained",
"No timed publisher phrase groundtruth: strictpoint scoring unavailable",
"All4 noanswer queries return files, no abstention improvement",
"Accepted publicSpeechTranscription values recorded, no native wire/stderr capture"
],
"production_policy_changed": false
}