Freeze normal-runtime speech timing regression without timestamp selection

tools/speech-eval/score_service_timing.py

@@ -0,0 +1,79 @@
+"""Adapter to unchanged publisher timing scorer; actual service top hit, no oracle."""
+import argparse
+from collections import Counter
+import hashlib
+import json
+from pathlib import Path
+
+from score import score as publisher_score
+from run import FIXTURE_SHA
+
+SOURCES = {"clip04", "music01", "music02", "sound01", "sound02"}
+
+
+def evaluate(report, fixture):
+    if report.get("complete") is not True or report.get("fixture_sha256") != FIXTURE_SHA:
+        raise ValueError("Incomplete/stale timing report")
+    case = report["timing"]
+    if {source["id"] for source in case["sources"]} != SOURCES or len(case["sources"]) != 5:
+        raise ValueError("All five competitors are mandatory")
+    expected_calls = Counter(window["pcm_sha256"] for source in case["sources"] for window in source["windows"])
+    actual_calls = Counter(call["pcm_sha256"] for call in case["native_calls"])
+    eligible = (case["comparison_eligible"] is True and expected_calls == actual_calls
+                and all("error" not in call for call in case["native_calls"]))
+    frozen = {query["id"]: query for query in fixture["queries"] if query["category"] == "video-speech"}
+    if len(frozen) != 8 or len(case["rows"]) != 8 or {row["query"]["id"] for row in case["rows"]} != set(frozen):
+        raise ValueError("All eight frozen queries required")
+    ranked = {}
+    diagnostics = []
+    for row in case["rows"]:
+        query = row["query"]
+        if query != frozen[query["id"]]:
+            raise ValueError("Frozen timing labels/text changed")
+        hits = row["hits"]
+        if not {hit["asset"] for hit in hits} <= SOURCES:
+            raise ValueError("Unexpected competitor")
+        first = hits[0] if hits else None
+        diagnostic = {"query": query["id"], "first_hit": first, "first_result_kind": None if not first else first["result"]["match_kind"], "returned_count": len(hits), "recognized_speech_first": False}
+        ranked[query["id"]] = []
+        if first and first["result"]["match_kind"] == "semantic_speech":
+            excerpt = first.get("verified_recognized_excerpt")
+            source = first["result"].get("source")
+            if not excerpt or not source or source["kind"] != "timestamp_ms" or excerpt["provenance"] != "local_speech_recognition" or excerpt["text"] != first["result"]["snippet"] or excerpt["source"] != source:
+                raise ValueError("Actual first recognized source citation missing")
+            ranked[query["id"]] = [{"asset": first["asset"], "start_ms": source["number"], "end_ms": excerpt["end_ms"]}]
+            diagnostic["recognized_speech_first"] = True
+        diagnostics.append(diagnostic)
+    scored = publisher_score(fixture, ranked) if eligible else None
+    file_top1 = sum(bool(row["hits"]) and row["hits"][0]["asset"] in row["query"]["expected"] for row in case["rows"])
+    return {"comparison_eligible": eligible, "strict_point_score": scored,
+            "correct_first_file": file_top1 if eligible else None,
+            "diagnostic_rows": diagnostics,
+            "scope": "Actual hybrid first result; raw media/non-speech origin remains an explicit recognized-speech timing failure; no lower hit replaces first; no carried-over full-source VAD score"}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("report", type=Path)
+    parser.add_argument("fixture", type=Path)
+    parser.add_argument("policy", type=Path)
+    parser.add_argument("output", type=Path)
+    args = parser.parse_args()
+    raw = args.report.read_bytes()
+    fixture_raw = args.fixture.read_bytes()
+    policy_raw = args.policy.read_bytes()
+    report = json.loads(raw)
+    policy = json.loads(policy_raw)
+    if hashlib.sha256(fixture_raw).hexdigest() != FIXTURE_SHA or hashlib.sha256(policy_raw).hexdigest() != report["timing_policy_sha256"]:
+        raise ValueError("Frozen fixture/policy changed")
+    scorer_raw = Path(__file__).with_name("score.py").read_bytes()
+    if policy["scoring_function_sha256"] != hashlib.sha256(scorer_raw).hexdigest():
+        raise ValueError("Publisher scoring function changed")
+    result = evaluate(report, json.loads(fixture_raw))
+    result.update(raw_report_sha256=hashlib.sha256(raw).hexdigest(), fixture_sha256=FIXTURE_SHA, timing_policy_sha256=report["timing_policy_sha256"], scoring_function_sha256=policy["scoring_function_sha256"])
+    with args.output.open("x", encoding="utf-8") as output:
+        json.dump(result, output, ensure_ascii=False, indent=2)
+
+
+if __name__ == "__main__":
+    main()