import unittest from score import metrics, temporal_hit, timestamp_distance class ScoringTests(unittest.TestCase): def test_wrong_file_cannot_receive_temporal_credit(self): query = dict(expected=["clip01"], intervals={"clip01": [[3, 5]]}) self.assertFalse(temporal_hit(dict(asset="clip02", start_s=3, end_s=5), query)) def test_coarse_audio_overlap_does_not_count_as_precise_timestamp(self): query = dict(expected=["speech"], intervals={"speech": [[13, 16]]}) candidate = dict(asset="speech", start_s=0, end_s=30) self.assertTrue(temporal_hit(candidate, query)) self.assertFalse(temporal_hit(candidate, query, point=True)) self.assertFalse(temporal_hit(dict(asset="speech", start_s=16, end_s=30), query)) self.assertEqual(timestamp_distance(candidate, query), 13) self.assertEqual(timestamp_distance(dict(asset="speech", start_s=15), query), 0) def test_visual_frame_cannot_receive_credit_for_a_spoken_topic(self): query = dict(expected=["movie"], intervals={"movie": [[10, 15]]}, required_modality="audio") candidate = dict(asset="movie", modality="image", start_s=12, end_s=14) self.assertFalse(temporal_hit(candidate, query)) self.assertIsNone(timestamp_distance(candidate, query)) def test_no_answers_and_unsupported_cases_do_not_inflate_recall(self): positive = dict(query=dict(expected=["good"], category="image-memory"), file_rankings=[dict(candidate=dict(asset="bad")), dict(candidate=dict(asset="good"))]) negative = dict(query=dict(expected=[], category="content-no-answer"), file_rankings=[dict(candidate=dict(asset="bad"))]) unsupported = dict(query=dict(expected=["good"], category="exact-metadata"), unsupported="metadata unavailable") result = metrics([positive, negative, unsupported]) self.assertEqual(result["positive_queries"], 1) self.assertEqual(result["recall_at1"], 0) self.assertEqual(result["recall_at5"], 1) self.assertEqual(result["mrr"], .5) self.assertEqual(result["no_answer_return_rate"], 1) self.assertEqual(result["unsupported_queries"], 1) if __name__ == "__main__": unittest.main()