From e7245fe15e122af51b320e9b7d0c9f5e25d6126d Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 12:17:37 +0200 Subject: [PATCH] Freeze calibration-derived document and code relevance hypothesis --- docs/RELEVANCE-CALIBRATION.md | 32 +++++++++ tools/relevance-eval/analyze_calibration.py | 68 +++++++++++++++++++ tools/relevance-eval/policies/cosine-v1.json | 14 ++++ .../test_calibration_analysis.py | 49 +++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 tools/relevance-eval/analyze_calibration.py create mode 100644 tools/relevance-eval/policies/cosine-v1.json create mode 100644 tools/relevance-eval/test_calibration_analysis.py diff --git a/docs/RELEVANCE-CALIBRATION.md b/docs/RELEVANCE-CALIBRATION.md index 1dae49f..31b836b 100644 --- a/docs/RELEVANCE-CALIBRATION.md +++ b/docs/RELEVANCE-CALIBRATION.md @@ -76,3 +76,35 @@ This ordered debug diagnostic used one CPU thread and the normal worker Lexical indexing took 19.9823 ms, semantic indexing 74,545.518 ms and retained query-session startup 7,433.3914 ms. These are diagnostic setup measurements, not native first-search latency or warm production p95. + +## Frozen diagnostic hypothesis, 2026-10-13 + +Before evaluating held-out requests, freeze +[`document-code-cosine-070-v1`](../tools/relevance-eval/policies/cosine-v1.json): +retain document/code semantic candidates with raw cosine **>= 0.70** before +hybrid fusion. Preserve lexical results and exact identifier priority unchanged. +This is a diagnostic hypothesis, not a deployed product change; media is excluded. +It is not a probability or a universal EmbeddingGemma threshold. + +The calibration analyzer accepts only the exact previously recorded report SHA, +rejects held-out inputs and nonfinite scores, and uses inclusive comparison. +Three structural tests passed in normal and optimized Python modes. + +| Scope | Minimum positive top cosine | Maximum absent top cosine | Retained positives / absent queries with retained neighbours | +| --- | --- | --- | --- | +| All files | 0.703227 | 0.693980 | 16/16 / 0/8 | +| Additional Code scope | 0.761020 | 0.656212 | 4/4 / 0/2 | + +The All files separation is only **0.009247**. The rounded 0.70 hypothesis +fits this small calibration sample; this narrow separation is a reason to +expect generalization failures, not to claim reliable abstention. Bilingual +pairs and repeated Code scope requests are correlated observations. +No margin, language-specific thresholds, task-specific tuning or model changes +are added to this version. Evaluate held-out once without adjusting this policy +from its outcomes; a failure requires recording the failure, not retuning it +against the same held-out sample. + +Policy SHA-256: +`df0a887b97dd7fda6aeff1b6c966fe7558521f40dbe1c911e91cfd312dca7461`. +Ignored analysis: `.searchi/qa/calibration-cosine-policy-v1.json`. +Held-out remains unevaluated at this freeze. Product ranking is unchanged. diff --git a/tools/relevance-eval/analyze_calibration.py b/tools/relevance-eval/analyze_calibration.py new file mode 100644 index 0000000..b3e0fd0 --- /dev/null +++ b/tools/relevance-eval/analyze_calibration.py @@ -0,0 +1,68 @@ +"""Inspect only the frozen calibration baseline; no held-out inputs accepted.""" +import argparse +import hashlib +import json +import math +from pathlib import Path + + +BASELINE_SHA256 = "56f42c523527dd748918bb8f17be6bd7818158227e692fccaf387d915f9d0a94" + + +def analyze(raw, policy): + if hashlib.sha256(raw).hexdigest() != BASELINE_SHA256: + raise ValueError("Expected exact frozen calibration baseline") + report = json.loads(raw) + if not report["complete"] or report["split"] != "calibration" or report["held_out_evaluated"]: + raise ValueError("Only complete calibration evidence is accepted") + if report["fixture_sha256"] != policy["fixture_sha256"]: + raise ValueError("Fixture differs from frozen hypothesis") + threshold = policy["semantic_cosine_minimum"] + if not math.isfinite(threshold) or not -1 <= threshold <= 1: + raise ValueError("Invalid cosine minimum") + scopes = {} + rows = [] + for row in report["rows"]: + if row["query"]["split"] != "calibration": + raise ValueError("Non-calibration query in report") + label = row["query"]["classification"] + if label not in ("positive", "absent_artifact"): + raise ValueError("Unsupported relevance label") + score = row["semantic_raw"][0]["score"] + if not math.isfinite(score) or not -1 <= score <= 1: + raise ValueError("Invalid raw semantic cosine") + accepted = score >= threshold + group = scopes.setdefault(row["scope"], { + "positive_scores": [], "absent_scores": [], + "retained_positive_top_results": 0, "absent_queries_with_retained_neighbors": 0, + }) + group["positive_scores" if label == "positive" else "absent_scores"].append(score) + if accepted: + group["retained_positive_top_results" if label == "positive" + else "absent_queries_with_retained_neighbors"] += 1 + rows.append({"id": row["query"]["id"], "scope": row["scope"], "label": label, + "top_cosine": score, "accepted_by_hypothesis": accepted}) + for group in scopes.values(): + positive = group.pop("positive_scores") + absent = group.pop("absent_scores") + group.update(positive_requests=len(positive), absent_requests=len(absent), + minimum_positive_top_cosine=min(positive), + maximum_absent_top_cosine=max(absent), + observed_separation=min(positive) - max(absent)) + return {"policy_id": policy["id"], "calibration_only": True, + "held_out_evaluated": False, "production_policy_changed": False, + "scopes": scopes, "rows": rows} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + policy_path = Path(__file__).parent / "policies/cosine-v1.json" + policy_bytes = policy_path.read_bytes() + result = analyze(args.report.read_bytes(), json.loads(policy_bytes)) + result["policy_sha256"] = hashlib.sha256(policy_bytes).hexdigest() + with args.output.open("x", encoding="utf-8") as stream: + json.dump(result, stream, indent=2) + print(json.dumps(result["scopes"], indent=2)) diff --git a/tools/relevance-eval/policies/cosine-v1.json b/tools/relevance-eval/policies/cosine-v1.json new file mode 100644 index 0000000..27418f5 --- /dev/null +++ b/tools/relevance-eval/policies/cosine-v1.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "id": "document-code-cosine-070-v1", + "status": "frozen diagnostic hypothesis, not production policy", + "calibration_report_sha256": "56f42c523527dd748918bb8f17be6bd7818158227e692fccaf387d915f9d0a94", + "fixture_sha256": "000047fd17accd315b6138093bbdc9f353cd877b4bf1958fbe336e4a6b10cd2e", + "semantic_cosine_minimum": 0.7, + "comparison": "greater_than_or_equal", + "scope": ["all_files", "code"], + "modality": "document and code text only", + "lexical_results": "preserve unchanged, including exact identifier priority", + "semantic_results": "retain only raw semantic cosine scores at or above the minimum before hybrid fusion", + "limitations": "Not a probability or universal model cutoff. Small bilingual paired corpus; media excluded. Do not tune this version after held-out evaluation." +} diff --git a/tools/relevance-eval/test_calibration_analysis.py b/tools/relevance-eval/test_calibration_analysis.py new file mode 100644 index 0000000..9ce659a --- /dev/null +++ b/tools/relevance-eval/test_calibration_analysis.py @@ -0,0 +1,49 @@ +import hashlib +import json +import unittest +from unittest.mock import patch + +import analyze_calibration as analysis + + +class CalibrationAnalysisTests(unittest.TestCase): + def fixture(self): + policy = {"id": "test", "fixture_sha256": "fixture", "semantic_cosine_minimum": 0.7} + rows = [{"scope": "all_files", "query": { + "id": label, "split": "calibration", "classification": label, + }, "semantic_raw": [{"score": score}]} for label, score in + [("positive", 0.7), ("absent_artifact", 0.69)]] + return policy, {"complete": True, "split": "calibration", + "held_out_evaluated": False, "fixture_sha256": "fixture", "rows": rows} + + def run_fixture(self, policy, report): + raw = json.dumps(report).encode() + with patch.object(analysis, "BASELINE_SHA256", hashlib.sha256(raw).hexdigest()): + return analysis.analyze(raw, policy) + + def test_boundary_is_inclusive_and_not_a_production_claim(self): + result = self.run_fixture(*self.fixture()) + self.assertTrue(result["rows"][0]["accepted_by_hypothesis"]) + self.assertFalse(result["rows"][1]["accepted_by_hypothesis"]) + self.assertFalse(result["production_policy_changed"]) + + def test_reject_altered_baseline_without_mocking_provenance(self): + policy, _ = self.fixture() + with self.assertRaisesRegex(ValueError, "exact frozen"): + analysis.analyze(b"{}", policy) + + def test_reject_held_out_and_nonfinite_scores(self): + for mutation in ("held_out", "nonfinite", "row_split"): + policy, report = self.fixture() + if mutation == "held_out": + report["held_out_evaluated"] = True + elif mutation == "row_split": + report["rows"][0]["query"]["split"] = "held_out" + else: + report["rows"][0]["semantic_raw"][0]["score"] = float("nan") + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + self.run_fixture(policy, report) + + +if __name__ == "__main__": + unittest.main()