52 lines
2.8 KiB
Python
52 lines
2.8 KiB
Python
|
import copy
|
||
|
|
import unittest
|
||
|
|
|
||
|
|
from run import FIXTURE_SHA
|
||
|
|
from score_service_timing import evaluate, SOURCES
|
||
|
|
|
||
|
|
|
||
|
|
class TimingTests(unittest.TestCase):
|
||
|
|
def inputs(self):
|
||
|
|
queries = [{"id": "q" + str(index), "text": "unchanged", "category": "video-speech", "expected": ["clip04"], "intervals": {"clip04": [[1.551, 3.603]]}} for index in range(8)]
|
||
|
|
fixture = {"queries": queries}
|
||
|
|
sources = [{"id": id, "windows": [{"pcm_sha256": id}]} for id in SOURCES]
|
||
|
|
calls = [{"pcm_sha256": source["id"], "transcription": {"passages": []}} for source in sources]
|
||
|
|
source = {"kind": "timestamp_ms", "number": 1551}
|
||
|
|
hit = {"asset": "clip04", "result": {"match_kind": "semantic_speech", "snippet": "real", "source": source}, "verified_recognized_excerpt": {"provenance": "local_speech_recognition", "source": source, "end_ms": 3603, "text": "real"}}
|
||
|
|
rows = [{"query": query, "hits": [copy.deepcopy(hit)]} for query in queries]
|
||
|
|
return {"complete": True, "fixture_sha256": FIXTURE_SHA, "timing": {"sources": sources, "native_calls": calls, "comparison_eligible": True, "rows": rows}}, fixture
|
||
|
|
|
||
|
|
def test_frozen_seconds_map_to_actual_integer_milliseconds(self):
|
||
|
|
report, fixture = self.inputs()
|
||
|
|
result = evaluate(report, fixture)
|
||
|
|
self.assertEqual(result["strict_point_score"]["point_hits"], 8)
|
||
|
|
self.assertEqual(result["strict_point_score"]["overlap_hits"], 8)
|
||
|
|
|
||
|
|
def test_lower_timestamp_inside_label_is_never_selected(self):
|
||
|
|
report, fixture = self.inputs()
|
||
|
|
for row in report["timing"]["rows"]:
|
||
|
|
original = copy.deepcopy(row["hits"][0])
|
||
|
|
row["hits"][0]["result"]["source"]["number"] = 0
|
||
|
|
row["hits"][0]["verified_recognized_excerpt"]["source"]["number"] = 0
|
||
|
|
row["hits"].append(original)
|
||
|
|
self.assertEqual(evaluate(report, fixture)["strict_point_score"]["point_hits"], 0)
|
||
|
|
|
||
|
|
def test_visual_first_result_is_not_relabelled_speech(self):
|
||
|
|
report, fixture = self.inputs()
|
||
|
|
for row in report["timing"]["rows"]:
|
||
|
|
row["hits"][0]["result"]["match_kind"] = "semantic_video"
|
||
|
|
self.assertEqual(evaluate(report, fixture)["strict_point_score"]["point_hits"], 0)
|
||
|
|
|
||
|
|
def test_dropped_no_speech_competitor_invalidates_numeric_score(self):
|
||
|
|
report, fixture = self.inputs()
|
||
|
|
report["timing"]["native_calls"].pop()
|
||
|
|
result = evaluate(report, fixture)
|
||
|
|
self.assertFalse(result["comparison_eligible"])
|
||
|
|
self.assertIsNone(result["strict_point_score"])
|
||
|
|
|
||
|
|
def test_modified_query_is_rejected(self):
|
||
|
|
report, fixture = self.inputs()
|
||
|
|
report = copy.deepcopy(report)
|
||
|
|
report["timing"]["rows"][0]["query"]["text"] = "changed"
|
||
|
|
with self.assertRaises(ValueError):
|
||
|
|
evaluate(report, fixture)
|