39 lines
2.2 KiB
Python
39 lines
2.2 KiB
Python
|
import unittest
|
||
|
|
from score import metrics, temporal_hit, timestamp_distance
|
||
|
|
|
||
|
|
|
||
|
|
class ScoringTests(unittest.TestCase):
|
||
|
|
def test_wrong_file_cannot_receive_temporal_credit(self):
|
||
|
|
query = dict(expected=["clip01"], intervals={"clip01": [[3, 5]]})
|
||
|
|
self.assertFalse(temporal_hit(dict(asset="clip02", start_s=3, end_s=5), query))
|
||
|
|
|
||
|
|
def test_coarse_audio_overlap_does_not_count_as_precise_timestamp(self):
|
||
|
|
query = dict(expected=["speech"], intervals={"speech": [[13, 16]]})
|
||
|
|
candidate = dict(asset="speech", start_s=0, end_s=30)
|
||
|
|
self.assertTrue(temporal_hit(candidate, query))
|
||
|
|
self.assertFalse(temporal_hit(candidate, query, point=True))
|
||
|
|
self.assertFalse(temporal_hit(dict(asset="speech", start_s=16, end_s=30), query))
|
||
|
|
self.assertEqual(timestamp_distance(candidate, query), 13)
|
||
|
|
self.assertEqual(timestamp_distance(dict(asset="speech", start_s=15), query), 0)
|
||
|
|
|
||
|
|
def test_visual_frame_cannot_receive_credit_for_a_spoken_topic(self):
|
||
|
|
query = dict(expected=["movie"], intervals={"movie": [[10, 15]]}, required_modality="audio")
|
||
|
|
candidate = dict(asset="movie", modality="image", start_s=12, end_s=14)
|
||
|
|
self.assertFalse(temporal_hit(candidate, query))
|
||
|
|
self.assertIsNone(timestamp_distance(candidate, query))
|
||
|
|
|
||
|
|
def test_no_answers_and_unsupported_cases_do_not_inflate_recall(self):
|
||
|
|
positive = dict(query=dict(expected=["good"], category="image-memory"), file_rankings=[dict(candidate=dict(asset="bad")), dict(candidate=dict(asset="good"))])
|
||
|
|
negative = dict(query=dict(expected=[], category="content-no-answer"), file_rankings=[dict(candidate=dict(asset="bad"))])
|
||
|
|
unsupported = dict(query=dict(expected=["good"], category="exact-metadata"), unsupported="metadata unavailable")
|
||
|
|
result = metrics([positive, negative, unsupported])
|
||
|
|
self.assertEqual(result["positive_queries"], 1)
|
||
|
|
self.assertEqual(result["recall_at1"], 0)
|
||
|
|
self.assertEqual(result["recall_at5"], 1)
|
||
|
|
self.assertEqual(result["mrr"], .5)
|
||
|
|
self.assertEqual(result["no_answer_return_rate"], 1)
|
||
|
|
self.assertEqual(result["unsupported_queries"], 1)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|