Freeze calibration-derived document and code relevance hypothesis

tools/relevance-eval/test_calibration_analysis.py

@@ -0,0 +1,49 @@
+import hashlib
+import json
+import unittest
+from unittest.mock import patch
+
+import analyze_calibration as analysis
+
+
+class CalibrationAnalysisTests(unittest.TestCase):
+    def fixture(self):
+        policy = {"id": "test", "fixture_sha256": "fixture", "semantic_cosine_minimum": 0.7}
+        rows = [{"scope": "all_files", "query": {
+            "id": label, "split": "calibration", "classification": label,
+        }, "semantic_raw": [{"score": score}]} for label, score in
+            [("positive", 0.7), ("absent_artifact", 0.69)]]
+        return policy, {"complete": True, "split": "calibration",
+                        "held_out_evaluated": False, "fixture_sha256": "fixture", "rows": rows}
+
+    def run_fixture(self, policy, report):
+        raw = json.dumps(report).encode()
+        with patch.object(analysis, "BASELINE_SHA256", hashlib.sha256(raw).hexdigest()):
+            return analysis.analyze(raw, policy)
+
+    def test_boundary_is_inclusive_and_not_a_production_claim(self):
+        result = self.run_fixture(*self.fixture())
+        self.assertTrue(result["rows"][0]["accepted_by_hypothesis"])
+        self.assertFalse(result["rows"][1]["accepted_by_hypothesis"])
+        self.assertFalse(result["production_policy_changed"])
+
+    def test_reject_altered_baseline_without_mocking_provenance(self):
+        policy, _ = self.fixture()
+        with self.assertRaisesRegex(ValueError, "exact frozen"):
+            analysis.analyze(b"{}", policy)
+
+    def test_reject_held_out_and_nonfinite_scores(self):
+        for mutation in ("held_out", "nonfinite", "row_split"):
+            policy, report = self.fixture()
+            if mutation == "held_out":
+                report["held_out_evaluated"] = True
+            elif mutation == "row_split":
+                report["rows"][0]["query"]["split"] = "held_out"
+            else:
+                report["rows"][0]["semantic_raw"][0]["score"] = float("nan")
+            with self.subTest(mutation=mutation), self.assertRaises(ValueError):
+                self.run_fixture(policy, report)
+
+
+if __name__ == "__main__":
+    unittest.main()