49 lines
2.0 KiB
Python
49 lines
2.0 KiB
Python
|
import hashlib
|
||
|
|
import json
|
||
|
|
import unittest
|
||
|
|
from unittest.mock import patch
|
||
|
|
|
||
|
|
import analyze_calibration as analysis
|
||
|
|
|
||
|
|
|
||
|
|
class CalibrationAnalysisTests(unittest.TestCase):
|
||
|
|
def fixture(self):
|
||
|
|
policy = {"id": "test", "fixture_sha256": "fixture", "semantic_cosine_minimum": 0.7}
|
||
|
|
rows = [{"scope": "all_files", "query": {
|
||
|
|
"id": label, "split": "calibration", "classification": label,
|
||
|
|
}, "semantic_raw": [{"score": score}]} for label, score in
|
||
|
|
[("positive", 0.7), ("absent_artifact", 0.69)]]
|
||
|
|
return policy, {"complete": True, "split": "calibration",
|
||
|
|
"held_out_evaluated": False, "fixture_sha256": "fixture", "rows": rows}
|
||
|
|
|
||
|
|
def run_fixture(self, policy, report):
|
||
|
|
raw = json.dumps(report).encode()
|
||
|
|
with patch.object(analysis, "BASELINE_SHA256", hashlib.sha256(raw).hexdigest()):
|
||
|
|
return analysis.analyze(raw, policy)
|
||
|
|
|
||
|
|
def test_boundary_is_inclusive_and_not_a_production_claim(self):
|
||
|
|
result = self.run_fixture(*self.fixture())
|
||
|
|
self.assertTrue(result["rows"][0]["accepted_by_hypothesis"])
|
||
|
|
self.assertFalse(result["rows"][1]["accepted_by_hypothesis"])
|
||
|
|
self.assertFalse(result["production_policy_changed"])
|
||
|
|
|
||
|
|
def test_reject_altered_baseline_without_mocking_provenance(self):
|
||
|
|
policy, _ = self.fixture()
|
||
|
|
with self.assertRaisesRegex(ValueError, "exact frozen"):
|
||
|
|
analysis.analyze(b"{}", policy)
|
||
|
|
|
||
|
|
def test_reject_held_out_and_nonfinite_scores(self):
|
||
|
|
for mutation in ("held_out", "nonfinite", "row_split"):
|
||
|
|
policy, report = self.fixture()
|
||
|
|
if mutation == "held_out":
|
||
|
|
report["held_out_evaluated"] = True
|
||
|
|
elif mutation == "row_split":
|
||
|
|
report["rows"][0]["query"]["split"] = "held_out"
|
||
|
|
else:
|
||
|
|
report["rows"][0]["semantic_raw"][0]["score"] = float("nan")
|
||
|
|
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
|
||
|
|
self.run_fixture(policy, report)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|