Freeze calibration-derived document and code relevance hypothesis

docs/RELEVANCE-CALIBRATION.md

@@ -76,3 +76,35 @@ This ordered debug diagnostic used one CPU thread and the normal worker
 Lexical indexing took 19.9823 ms, semantic indexing 74,545.518 ms and retained
 query-session startup 7,433.3914 ms. These are diagnostic setup measurements,
 not native first-search latency or warm production p95.
+
+## Frozen diagnostic hypothesis, 2026-10-13
+
+Before evaluating held-out requests, freeze
+[`document-code-cosine-070-v1`](../tools/relevance-eval/policies/cosine-v1.json):
+retain document/code semantic candidates with raw cosine **>= 0.70** before
+hybrid fusion. Preserve lexical results and exact identifier priority unchanged.
+This is a diagnostic hypothesis, not a deployed product change; media is excluded.
+It is not a probability or a universal EmbeddingGemma threshold.
+
+The calibration analyzer accepts only the exact previously recorded report SHA,
+rejects held-out inputs and nonfinite scores, and uses inclusive comparison.
+Three structural tests passed in normal and optimized Python modes.
+
+| Scope | Minimum positive top cosine | Maximum absent top cosine | Retained positives / absent queries with retained neighbours |
+| --- | --- | --- | --- |
+| All files | 0.703227 | 0.693980 | 16/16 / 0/8 |
+| Additional Code scope | 0.761020 | 0.656212 | 4/4 / 0/2 |
+
+The All files separation is only **0.009247**. The rounded 0.70 hypothesis
+fits this small calibration sample; this narrow separation is a reason to
+expect generalization failures, not to claim reliable abstention. Bilingual
+pairs and repeated Code scope requests are correlated observations.
+No margin, language-specific thresholds, task-specific tuning or model changes
+are added to this version. Evaluate held-out once without adjusting this policy
+from its outcomes; a failure requires recording the failure, not retuning it
+against the same held-out sample.
+
+Policy SHA-256:
+`df0a887b97dd7fda6aeff1b6c966fe7558521f40dbe1c911e91cfd312dca7461`.
+Ignored analysis: `.searchi/qa/calibration-cosine-policy-v1.json`.
+Held-out remains unevaluated at this freeze. Product ranking is unchanged.

tools/relevance-eval/analyze_calibration.py

@@ -0,0 +1,68 @@
+"""Inspect only the frozen calibration baseline; no held-out inputs accepted."""
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+
+
+BASELINE_SHA256 = "56f42c523527dd748918bb8f17be6bd7818158227e692fccaf387d915f9d0a94"
+
+
+def analyze(raw, policy):
+    if hashlib.sha256(raw).hexdigest() != BASELINE_SHA256:
+        raise ValueError("Expected exact frozen calibration baseline")
+    report = json.loads(raw)
+    if not report["complete"] or report["split"] != "calibration" or report["held_out_evaluated"]:
+        raise ValueError("Only complete calibration evidence is accepted")
+    if report["fixture_sha256"] != policy["fixture_sha256"]:
+        raise ValueError("Fixture differs from frozen hypothesis")
+    threshold = policy["semantic_cosine_minimum"]
+    if not math.isfinite(threshold) or not -1 <= threshold <= 1:
+        raise ValueError("Invalid cosine minimum")
+    scopes = {}
+    rows = []
+    for row in report["rows"]:
+        if row["query"]["split"] != "calibration":
+            raise ValueError("Non-calibration query in report")
+        label = row["query"]["classification"]
+        if label not in ("positive", "absent_artifact"):
+            raise ValueError("Unsupported relevance label")
+        score = row["semantic_raw"][0]["score"]
+        if not math.isfinite(score) or not -1 <= score <= 1:
+            raise ValueError("Invalid raw semantic cosine")
+        accepted = score >= threshold
+        group = scopes.setdefault(row["scope"], {
+            "positive_scores": [], "absent_scores": [],
+            "retained_positive_top_results": 0, "absent_queries_with_retained_neighbors": 0,
+        })
+        group["positive_scores" if label == "positive" else "absent_scores"].append(score)
+        if accepted:
+            group["retained_positive_top_results" if label == "positive"
+                  else "absent_queries_with_retained_neighbors"] += 1
+        rows.append({"id": row["query"]["id"], "scope": row["scope"], "label": label,
+                     "top_cosine": score, "accepted_by_hypothesis": accepted})
+    for group in scopes.values():
+        positive = group.pop("positive_scores")
+        absent = group.pop("absent_scores")
+        group.update(positive_requests=len(positive), absent_requests=len(absent),
+                     minimum_positive_top_cosine=min(positive),
+                     maximum_absent_top_cosine=max(absent),
+                     observed_separation=min(positive) - max(absent))
+    return {"policy_id": policy["id"], "calibration_only": True,
+            "held_out_evaluated": False, "production_policy_changed": False,
+            "scopes": scopes, "rows": rows}
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("report", type=Path)
+    parser.add_argument("output", type=Path)
+    args = parser.parse_args()
+    policy_path = Path(__file__).parent / "policies/cosine-v1.json"
+    policy_bytes = policy_path.read_bytes()
+    result = analyze(args.report.read_bytes(), json.loads(policy_bytes))
+    result["policy_sha256"] = hashlib.sha256(policy_bytes).hexdigest()
+    with args.output.open("x", encoding="utf-8") as stream:
+        json.dump(result, stream, indent=2)
+    print(json.dumps(result["scopes"], indent=2))

tools/relevance-eval/policies/cosine-v1.json

@@ -0,0 +1,14 @@
+{
+  "version": 1,
+  "id": "document-code-cosine-070-v1",
+  "status": "frozen diagnostic hypothesis, not production policy",
+  "calibration_report_sha256": "56f42c523527dd748918bb8f17be6bd7818158227e692fccaf387d915f9d0a94",
+  "fixture_sha256": "000047fd17accd315b6138093bbdc9f353cd877b4bf1958fbe336e4a6b10cd2e",
+  "semantic_cosine_minimum": 0.7,
+  "comparison": "greater_than_or_equal",
+  "scope": ["all_files", "code"],
+  "modality": "document and code text only",
+  "lexical_results": "preserve unchanged, including exact identifier priority",
+  "semantic_results": "retain only raw semantic cosine scores at or above the minimum before hybrid fusion",
+  "limitations": "Not a probability or universal model cutoff. Small bilingual paired corpus; media excluded. Do not tune this version after held-out evaluation."
+}

tools/relevance-eval/test_calibration_analysis.py

@@ -0,0 +1,49 @@
+import hashlib
+import json
+import unittest
+from unittest.mock import patch
+
+import analyze_calibration as analysis
+
+
+class CalibrationAnalysisTests(unittest.TestCase):
+    def fixture(self):
+        policy = {"id": "test", "fixture_sha256": "fixture", "semantic_cosine_minimum": 0.7}
+        rows = [{"scope": "all_files", "query": {
+            "id": label, "split": "calibration", "classification": label,
+        }, "semantic_raw": [{"score": score}]} for label, score in
+            [("positive", 0.7), ("absent_artifact", 0.69)]]
+        return policy, {"complete": True, "split": "calibration",
+                        "held_out_evaluated": False, "fixture_sha256": "fixture", "rows": rows}
+
+    def run_fixture(self, policy, report):
+        raw = json.dumps(report).encode()
+        with patch.object(analysis, "BASELINE_SHA256", hashlib.sha256(raw).hexdigest()):
+            return analysis.analyze(raw, policy)
+
+    def test_boundary_is_inclusive_and_not_a_production_claim(self):
+        result = self.run_fixture(*self.fixture())
+        self.assertTrue(result["rows"][0]["accepted_by_hypothesis"])
+        self.assertFalse(result["rows"][1]["accepted_by_hypothesis"])
+        self.assertFalse(result["production_policy_changed"])
+
+    def test_reject_altered_baseline_without_mocking_provenance(self):
+        policy, _ = self.fixture()
+        with self.assertRaisesRegex(ValueError, "exact frozen"):
+            analysis.analyze(b"{}", policy)
+
+    def test_reject_held_out_and_nonfinite_scores(self):
+        for mutation in ("held_out", "nonfinite", "row_split"):
+            policy, report = self.fixture()
+            if mutation == "held_out":
+                report["held_out_evaluated"] = True
+            elif mutation == "row_split":
+                report["rows"][0]["query"]["split"] = "held_out"
+            else:
+                report["rows"][0]["semantic_raw"][0]["score"] = float("nan")
+            with self.subTest(mutation=mutation), self.assertRaises(ValueError):
+                self.run_fixture(policy, report)
+
+
+if __name__ == "__main__":
+    unittest.main()