79 lines
4.4 KiB
Python
79 lines
4.4 KiB
Python
|
"""Adapter to unchanged publisher timing scorer; actual service top hit, no oracle."""
|
||
|
|
import argparse
|
||
|
|
from collections import Counter
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from score import score as publisher_score
|
||
|
|
from run import FIXTURE_SHA
|
||
|
|
|
||
|
|
SOURCES = {"clip04", "music01", "music02", "sound01", "sound02"}
|
||
|
|
|
||
|
|
|
||
|
|
def evaluate(report, fixture):
|
||
|
|
if report.get("complete") is not True or report.get("fixture_sha256") != FIXTURE_SHA:
|
||
|
|
raise ValueError("Incomplete/stale timing report")
|
||
|
|
case = report["timing"]
|
||
|
|
if {source["id"] for source in case["sources"]} != SOURCES or len(case["sources"]) != 5:
|
||
|
|
raise ValueError("All five competitors are mandatory")
|
||
|
|
expected_calls = Counter(window["pcm_sha256"] for source in case["sources"] for window in source["windows"])
|
||
|
|
actual_calls = Counter(call["pcm_sha256"] for call in case["native_calls"])
|
||
|
|
eligible = (case["comparison_eligible"] is True and expected_calls == actual_calls
|
||
|
|
and all("error" not in call for call in case["native_calls"]))
|
||
|
|
frozen = {query["id"]: query for query in fixture["queries"] if query["category"] == "video-speech"}
|
||
|
|
if len(frozen) != 8 or len(case["rows"]) != 8 or {row["query"]["id"] for row in case["rows"]} != set(frozen):
|
||
|
|
raise ValueError("All eight frozen queries required")
|
||
|
|
ranked = {}
|
||
|
|
diagnostics = []
|
||
|
|
for row in case["rows"]:
|
||
|
|
query = row["query"]
|
||
|
|
if query != frozen[query["id"]]:
|
||
|
|
raise ValueError("Frozen timing labels/text changed")
|
||
|
|
hits = row["hits"]
|
||
|
|
if not {hit["asset"] for hit in hits} <= SOURCES:
|
||
|
|
raise ValueError("Unexpected competitor")
|
||
|
|
first = hits[0] if hits else None
|
||
|
|
diagnostic = {"query": query["id"], "first_hit": first, "first_result_kind": None if not first else first["result"]["match_kind"], "returned_count": len(hits), "recognized_speech_first": False}
|
||
|
|
ranked[query["id"]] = []
|
||
|
|
if first and first["result"]["match_kind"] == "semantic_speech":
|
||
|
|
excerpt = first.get("verified_recognized_excerpt")
|
||
|
|
source = first["result"].get("source")
|
||
|
|
if not excerpt or not source or source["kind"] != "timestamp_ms" or excerpt["provenance"] != "local_speech_recognition" or excerpt["text"] != first["result"]["snippet"] or excerpt["source"] != source:
|
||
|
|
raise ValueError("Actual first recognized source citation missing")
|
||
|
|
ranked[query["id"]] = [{"asset": first["asset"], "start_ms": source["number"], "end_ms": excerpt["end_ms"]}]
|
||
|
|
diagnostic["recognized_speech_first"] = True
|
||
|
|
diagnostics.append(diagnostic)
|
||
|
|
scored = publisher_score(fixture, ranked) if eligible else None
|
||
|
|
file_top1 = sum(bool(row["hits"]) and row["hits"][0]["asset"] in row["query"]["expected"] for row in case["rows"])
|
||
|
|
return {"comparison_eligible": eligible, "strict_point_score": scored,
|
||
|
|
"correct_first_file": file_top1 if eligible else None,
|
||
|
|
"diagnostic_rows": diagnostics,
|
||
|
|
"scope": "Actual hybrid first result; raw media/non-speech origin remains an explicit recognized-speech timing failure; no lower hit replaces first; no carried-over full-source VAD score"}
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
parser.add_argument("report", type=Path)
|
||
|
|
parser.add_argument("fixture", type=Path)
|
||
|
|
parser.add_argument("policy", type=Path)
|
||
|
|
parser.add_argument("output", type=Path)
|
||
|
|
args = parser.parse_args()
|
||
|
|
raw = args.report.read_bytes()
|
||
|
|
fixture_raw = args.fixture.read_bytes()
|
||
|
|
policy_raw = args.policy.read_bytes()
|
||
|
|
report = json.loads(raw)
|
||
|
|
policy = json.loads(policy_raw)
|
||
|
|
if hashlib.sha256(fixture_raw).hexdigest() != FIXTURE_SHA or hashlib.sha256(policy_raw).hexdigest() != report["timing_policy_sha256"]:
|
||
|
|
raise ValueError("Frozen fixture/policy changed")
|
||
|
|
scorer_raw = Path(__file__).with_name("score.py").read_bytes()
|
||
|
|
if policy["scoring_function_sha256"] != hashlib.sha256(scorer_raw).hexdigest():
|
||
|
|
raise ValueError("Publisher scoring function changed")
|
||
|
|
result = evaluate(report, json.loads(fixture_raw))
|
||
|
|
result.update(raw_report_sha256=hashlib.sha256(raw).hexdigest(), fixture_sha256=FIXTURE_SHA, timing_policy_sha256=report["timing_policy_sha256"], scoring_function_sha256=policy["scoring_function_sha256"])
|
||
|
|
with args.output.open("x", encoding="utf-8") as output:
|
||
|
|
json.dump(result, output, ensure_ascii=False, indent=2)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|