Freeze independently reviewed relevance calibration corpus

fixtures/relevance-v2/test_verify.py

@@ -0,0 +1,82 @@
+"""Integrity regressions use isolated copies and execute the verifier under -O."""
+
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+class IntegrityTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory(prefix="searchi-relevance-integrity-")
+        self.root = Path(self.temporary.name) / "fixture"
+        shutil.copytree(Path(__file__).resolve().parent, self.root)
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def write(self, name, data):
+        (self.root / name).write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
+
+    def read(self, name):
+        return json.loads((self.root / name).read_text(encoding="utf-8"))
+
+    def run_verifier(self):
+        return subprocess.run([sys.executable, "-O", str(self.root / "verify.py")],
+                              capture_output=True, text=True, timeout=10)
+
+    def test_valid_frozen_fixture_under_optimization(self):
+        result = self.run_verifier()
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_git_preserves_all_locked_bytes_with_text_normalization(self):
+        repository = self.root.parent
+        (repository / ".gitattributes").write_text("* text=auto eol=lf\n", encoding="utf-8")
+        subprocess.run(["git", "init", "--quiet", str(repository)], check=True,
+                       capture_output=True, timeout=10)
+        subprocess.run(["git", "-c", "core.autocrlf=true", "add", "."], cwd=repository,
+                       check=True, capture_output=True, timeout=10)
+        for name, expected in self.read("fixture.lock.json")["sha256"].items():
+            staged = subprocess.check_output(["git", "show", ":fixture/" + name],
+                                             cwd=repository, timeout=10)
+            self.assertEqual(hashlib.sha256(staged).hexdigest(), expected, name)
+
+    def test_same_length_source_corruption_rejected(self):
+        source = self.root / "sources/bicycle-a.md"
+        original = source.read_bytes()
+        source.write_bytes(b"!" + original[1:])
+        self.assertNotEqual(self.run_verifier().returncode, 0)
+
+    def test_duplicate_asset_rejected(self):
+        for name in ["inventory.json", "blind-review-input.json"]:
+            data = self.read(name)
+            data["assets"].append(data["assets"][0])
+            self.write(name, data)
+        self.assertNotEqual(self.run_verifier().returncode, 0)
+
+    def test_bilingual_split_leak_rejected(self):
+        data = self.read("blind-review-input.json")
+        data["queries"][0]["split"] = "held_out"
+        self.write("blind-review-input.json", data)
+        self.assertNotEqual(self.run_verifier().returncode, 0)
+
+    def test_repository_topic_split_leak_rejected(self):
+        for name in ["inventory.json", "blind-review-input.json"]:
+            data = self.read(name)
+            next(asset for asset in data["assets"] if asset["id"] == "bicycle-a")["split"] = "held_out"
+            self.write(name, data)
+        self.assertNotEqual(self.run_verifier().returncode, 0)
+
+    def test_stale_lock_rejected(self):
+        lock = self.read("fixture.lock.json")
+        lock["sha256"]["fixture.json"] = "0" * 64
+        self.write("fixture.lock.json", lock)
+        self.assertNotEqual(self.run_verifier().returncode, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()