{ "source_commit": "2f9190c", "evaluator_commit": "48f11de", "service_source_commit": "dab7034", "fixture_sha256": "e3a850d7a2385e250408f57fe51f4620ce80a20f8625ced42205f02cc46d517d", "prepared_sha256": "085f34f0af863656a28e7a905746973a54d1123bb07053541e09c8af0a58bb3f", "binary_sha256": "7aa75b7765b7567a3f6761c6b72449092d5adab7c3a59aaa8e55ed622340e910", "speech_worker_sha256": "299c1b784eac46b31d228453a399e1378f327fc9ed5e34ab950c804da33e433a", "speech_inventory_sha256": "4904995685a5e55e0a790d9cc409dc8458e39c30cda4097c5d70bf75cb5c976c", "google_worker_sha256": "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73", "base_sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe", "vad_sha256": "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987", "speech_policy": "live-window-vad-dtw-v1;silero-v6.2.0;base;30s;vad-reset;threshold0.5;minspeech250ms;minsilence100ms;maxspeechFLT_MAX;pad30ms;vad-overlap0.1s;source-sample-intersections;beam5;bestof5;temp0;tempinc0.2;entropy2.4;logprob-1;nospeech0.6;suppressblank;no-suppress-nst;auto-language;no-context;dtw-base;no-flash;contiguous-intervals;first-last-token-points;exact-utf8", "artifact_hashes": { "preflight.json": { "bytes": 2075, "sha256": "63a0a615d37766b852c34473712156e950aeeb5a472a709bff6be00b7b26cb7a" }, "report.json": { "bytes": 174868, "sha256": "79ecb75d31c0193642c81862fd5476ad47adca518d4959e436bdd472cc281798" }, "scored.json": { "bytes": 5608, "sha256": "21d9bc84410cc63b96e2b575d99cd3c09834dfd9d2db2d0f356159df12e048e9" } }, "results": { "comparison_eligible": true, "strict_point_score_eligible": false, "positive_queries": 6, "top1": 6, "recall_at_5": 6, "mrr": 1.0, "negative_queries": 4, "negative_returned_any": 4, "native_passage_counts": { "music-control": 0, "silence-control": 0, "speech-de": 3, "speech-en": 4 }, "negative_gate_acceptance": true, "reference_concept_substring_coverage": { "speech-en": { "free": true, "equal": true, "dignity": true, "rights": true, "reason": true, "conscience": true, "brotherhood": true }, "speech-de": { "frei": true, "gleich": true, "Würde": true, "Rechten": true, "Vernunft": true, "Gewissen": true, "Brüderlichkeit": true } }, "rows": [ { "id": "equal-en", "language": "en", "positive": true, "first_relevant_rank": 1, "returned_count": 4, "first_hit": { "asset": "speech-de", "match_kind": "semantic_speech", "snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.", "source": { "kind": "timestamp_ms", "number": 7430 } } }, { "id": "equal-de", "language": "de", "positive": true, "first_relevant_rank": 1, "returned_count": 2, "first_hit": { "asset": "speech-de", "match_kind": "semantic_speech", "snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.", "source": { "kind": "timestamp_ms", "number": 7430 } } }, { "id": "reason-en", "language": "en", "positive": true, "first_relevant_rank": 1, "returned_count": 2, "first_hit": { "asset": "speech-en", "match_kind": "semantic_speech", "snippet": " they are endowed with reason and conscience.", "source": { "kind": "timestamp_ms", "number": 14730 } } }, { "id": "reason-de", "language": "de", "positive": true, "first_relevant_rank": 1, "returned_count": 4, "first_hit": { "asset": "speech-de", "match_kind": "semantic_speech", "snippet": " Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geisterbrüderlichkeit begegnen.", "source": { "kind": "timestamp_ms", "number": 11100 } } }, { "id": "solidarity-en", "language": "en", "positive": true, "first_relevant_rank": 1, "returned_count": 4, "first_hit": { "asset": "speech-en", "match_kind": "semantic_speech", "snippet": " and should act towards one another in a spirit of brotherhood.", "source": { "kind": "timestamp_ms", "number": 18430 } } }, { "id": "solidarity-de", "language": "de", "positive": true, "first_relevant_rank": 1, "returned_count": 4, "first_hit": { "asset": "speech-de", "match_kind": "semantic_speech", "snippet": " Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geisterbrüderlichkeit begegnen.", "source": { "kind": "timestamp_ms", "number": 11100 } } }, { "id": "absent-mars-en", "language": "en", "positive": false, "first_relevant_rank": null, "returned_count": 4, "first_hit": { "asset": "speech-en", "match_kind": "semantic_speech", "snippet": " they are endowed with reason and conscience.", "source": { "kind": "timestamp_ms", "number": 14730 } } }, { "id": "absent-mars-de", "language": "de", "positive": false, "first_relevant_rank": null, "returned_count": 4, "first_hit": { "asset": "speech-en", "match_kind": "semantic_speech", "snippet": " they are endowed with reason and conscience.", "source": { "kind": "timestamp_ms", "number": 14730 } } }, { "id": "absent-salary-en", "language": "en", "positive": false, "first_relevant_rank": null, "returned_count": 4, "first_hit": { "asset": "speech-en", "match_kind": "semantic_speech", "snippet": " they are endowed with reason and conscience.", "source": { "kind": "timestamp_ms", "number": 14730 } } }, { "id": "absent-salary-de", "language": "de", "positive": false, "first_relevant_rank": null, "returned_count": 4, "first_hit": { "asset": "speech-de", "match_kind": "semantic_speech", "snippet": " Alle Menschen sind frei und gleich an Würde und Rechten geboren.", "source": { "kind": "timestamp_ms", "number": 7430 } } } ] }, "cases": { "core": { "expected_native_calls": 4, "actual_native_calls": 4, "index_report": { "audio_inference_calls": 4, "chunks": 11, "document_inference_calls": 0, "encoded_sources": 8, "failures": [], "generation": 1, "image_inference_calls": 0, "media_failures": 0, "media_files": 4, "model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1:speech-v1-422af6d2bbbaaa3e943700c10035116aefcee24f35cd909bf33646bc41e00980", "passages": 0, "resumed": false, "reused_chunks": 0, "reused_sources": 0, "speech_chunks": 7, "speech_failures": [], "speech_metrics": { "admission_us": 0, "asr_us": 61574700, "fallback_us": 0, "google_embedding_calls": 7, "load_us": 209078, "transport_us": 212365, "vad_us": 216972 }, "speech_requested": true, "speech_reused_sources": 0, "speech_windows": 4 }, "indexing_ms": 127611.1518, "speech_load_verify_ms": 3330.588, "google_index_load_verify_ms": 7431.4775, "google_query_load_verify_ms": 7615.0823, "query_warmup_ms": 128.039, "windows": [ { "id": "speech-en", "windows": [ { "asset": "speech-en", "duration_ms": 22512, "pcm_sha256": "56948226ac5a0ad4b70523a856dad0851929c7067957f339e9a5b30fc9b369ce", "samples": 360192, "start_ms": 0 } ] }, { "id": "speech-de", "windows": [ { "asset": "speech-de", "duration_ms": 16219, "pcm_sha256": "136a1c03298a55f0ea7869e84a4684d7bd9e9fca4477b69ba860e7e300712ca2", "samples": 259504, "start_ms": 0 } ] }, { "id": "music-control", "windows": [ { "asset": "music-control", "duration_ms": 30000, "pcm_sha256": "a1fd7233bb4f9088688cfb36e449fb68b8bd5cdc70f2d193371ed9587b905193", "samples": 480000, "start_ms": 0 } ] }, { "id": "silence-control", "windows": [ { "asset": "silence-control", "duration_ms": 30000, "pcm_sha256": "124617c1f65e92d3bc895fbd869e4bb16a30754b198f59e6e973949b9aaa1b01", "samples": 480000, "start_ms": 0 } ] } ] }, "seams": { "expected_native_calls": 4, "actual_native_calls": 4, "index_report": { "audio_inference_calls": 4, "chunks": 24, "document_inference_calls": 0, "encoded_sources": 4, "failures": [], "generation": 1, "image_inference_calls": 0, "media_failures": 0, "media_files": 2, "model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1:speech-v1-422af6d2bbbaaa3e943700c10035116aefcee24f35cd909bf33646bc41e00980", "passages": 0, "resumed": false, "reused_chunks": 0, "reused_sources": 0, "speech_chunks": 20, "speech_failures": [], "speech_metrics": { "admission_us": 0, "asr_us": 121231168, "fallback_us": 0, "google_embedding_calls": 20, "load_us": 213591, "transport_us": 109395, "vad_us": 187061 }, "speech_requested": true, "speech_reused_sources": 0, "speech_windows": 4 }, "indexing_ms": 163367.8024, "speech_load_verify_ms": 3011.0661, "google_index_load_verify_ms": 7365.343099999999, "google_query_load_verify_ms": 7039.3844, "query_warmup_ms": 115.2243, "windows": [ { "id": "seam-en", "windows": [ { "asset": "seam-en", "duration_ms": 30000, "pcm_sha256": "7b9e755453e90f67f4ee9f1a6c4aaa5048225faeb266e9e187c890baac6a7138", "samples": 480000, "start_ms": 0 }, { "asset": "seam-en", "duration_ms": 12024, "pcm_sha256": "40b1bac0855bee11643dcce5d2e38801c5d3533993bd035f6a3c70ef219f180f", "samples": 192384, "start_ms": 28000 } ] }, { "id": "seam-de", "windows": [ { "asset": "seam-de", "duration_ms": 30000, "pcm_sha256": "9b77a526b62d1f473a014626eb312f995b3dd4b788c96bc31981edd243c50eea", "samples": 480000, "start_ms": 0 }, { "asset": "seam-de", "duration_ms": 10657, "pcm_sha256": "5d5c5988ab40601479b72498e7a63dbfedfe0567572597da52811b63c0a5d23b", "samples": 170512, "start_ms": 28000 } ] } ] } }, "limitations": [ "InternalCPUv3 runtime; normalAuto/GPU behavior not evaluated", "Native normalAuto compilation jobs1 overlapped sharedVM timings; no controlled hardware throughput claim", "Translated content correlated; concept substrings are not WER or exact transcript fidelity", "German Geisterbrüderlichkeit and Artikel1s recognized incorrectly; seam partial/duplicate text retained", "No timed publisher phrase groundtruth: strictpoint scoring unavailable", "All4 noanswer queries return files, no abstention improvement", "Accepted publicSpeechTranscription values recorded, no native wire/stderr capture" ], "production_policy_changed": false }