"""Adapter to unchanged publisher timing scorer; actual service top hit, no oracle.""" import argparse from collections import Counter import hashlib import json from pathlib import Path from score import score as publisher_score from run import FIXTURE_SHA SOURCES = {"clip04", "music01", "music02", "sound01", "sound02"} def evaluate(report, fixture): if report.get("complete") is not True or report.get("fixture_sha256") != FIXTURE_SHA: raise ValueError("Incomplete/stale timing report") case = report["timing"] if {source["id"] for source in case["sources"]} != SOURCES or len(case["sources"]) != 5: raise ValueError("All five competitors are mandatory") expected_calls = Counter(window["pcm_sha256"] for source in case["sources"] for window in source["windows"]) actual_calls = Counter(call["pcm_sha256"] for call in case["native_calls"]) eligible = (case["comparison_eligible"] is True and expected_calls == actual_calls and all("error" not in call for call in case["native_calls"])) frozen = {query["id"]: query for query in fixture["queries"] if query["category"] == "video-speech"} if len(frozen) != 8 or len(case["rows"]) != 8 or {row["query"]["id"] for row in case["rows"]} != set(frozen): raise ValueError("All eight frozen queries required") ranked = {} diagnostics = [] for row in case["rows"]: query = row["query"] if query != frozen[query["id"]]: raise ValueError("Frozen timing labels/text changed") hits = row["hits"] if not {hit["asset"] for hit in hits} <= SOURCES: raise ValueError("Unexpected competitor") first = hits[0] if hits else None diagnostic = {"query": query["id"], "first_hit": first, "first_result_kind": None if not first else first["result"]["match_kind"], "returned_count": len(hits), "recognized_speech_first": False} ranked[query["id"]] = [] if first and first["result"]["match_kind"] == "semantic_speech": excerpt = first.get("verified_recognized_excerpt") source = first["result"].get("source") if not excerpt or not source or source["kind"] != "timestamp_ms" or excerpt["provenance"] != "local_speech_recognition" or excerpt["text"] != first["result"]["snippet"] or excerpt["source"] != source: raise ValueError("Actual first recognized source citation missing") ranked[query["id"]] = [{"asset": first["asset"], "start_ms": source["number"], "end_ms": excerpt["end_ms"]}] diagnostic["recognized_speech_first"] = True diagnostics.append(diagnostic) scored = publisher_score(fixture, ranked) if eligible else None file_top1 = sum(bool(row["hits"]) and row["hits"][0]["asset"] in row["query"]["expected"] for row in case["rows"]) return {"comparison_eligible": eligible, "strict_point_score": scored, "correct_first_file": file_top1 if eligible else None, "diagnostic_rows": diagnostics, "scope": "Actual hybrid first result; raw media/non-speech origin remains an explicit recognized-speech timing failure; no lower hit replaces first; no carried-over full-source VAD score"} def main(): parser = argparse.ArgumentParser() parser.add_argument("report", type=Path) parser.add_argument("fixture", type=Path) parser.add_argument("policy", type=Path) parser.add_argument("output", type=Path) args = parser.parse_args() raw = args.report.read_bytes() fixture_raw = args.fixture.read_bytes() policy_raw = args.policy.read_bytes() report = json.loads(raw) policy = json.loads(policy_raw) if hashlib.sha256(fixture_raw).hexdigest() != FIXTURE_SHA or hashlib.sha256(policy_raw).hexdigest() != report["timing_policy_sha256"]: raise ValueError("Frozen fixture/policy changed") scorer_raw = Path(__file__).with_name("score.py").read_bytes() if policy["scoring_function_sha256"] != hashlib.sha256(scorer_raw).hexdigest(): raise ValueError("Publisher scoring function changed") result = evaluate(report, json.loads(fixture_raw)) result.update(raw_report_sha256=hashlib.sha256(raw).hexdigest(), fixture_sha256=FIXTURE_SHA, timing_policy_sha256=report["timing_policy_sha256"], scoring_function_sha256=policy["scoring_function_sha256"]) with args.output.open("x", encoding="utf-8") as output: json.dump(result, output, ensure_ascii=False, indent=2) if __name__ == "__main__": main()