Record unseen music regressions and actual local reranker fidelity and cost

tools/rerank-eval/README.md

@@ -0,0 +1,44 @@
+# Local cross-encoder diagnostic
+
+This standalone research tool never modifies production search or its Google
+EmbeddingGemma 2 model. Do not confuse the cross-encoder's name with the historical
+MiniLM embedding experiment. Publisher weights stay under ignored `.searchi` data.
+
+Pinned publisher revision, asset bytes/SHA-256 and licence provenance are in
+`model-manifest.json`; the pre-inference contract is in `compatibility-plan.json`.
+The existing CPU Python environment can execute it without installing Torch or
+changing production dependencies:
+
+```powershell
+python tools/rerank-eval/download.py
+$env:SEARCHI_FORCE_CPU = '1'
+& tools/semantic-proof/.venv/Scripts/python.exe tools/rerank-eval/compatibility.py --output .searchi/rerank-eval/compatibility-v1.json
+```
+
+The runner verifies all assets, rejects changed frozen inputs, enables only CPU
+Execution Provider with one thread, bounds five pairs to 256 tokens including
+special tokens and at most 64 query tokens, checks finite one-logit-per-pair
+outputs, and records serial/batch parity and twenty warm timing samples. It creates
+new reports atomically and refuses overwriting prior evidence. A failed parity
+contract writes the observed report then exits nonzero. Never weaken its frozen
+tolerance after observing output.
+
+The first execution failed serial/batch logit parity; see
+[recorded result](../../docs/LOCAL-RERANKING.md). It is not a product integration,
+relevance calibration or full search performance benchmark. Scores are not
+probabilities that the requested artifact exists. Any further experimental input
+or scoring policy needs a separate version and output path.
+
+The separate unquantized follow-up preserves the first failed evidence:
+
+```powershell
+python tools/rerank-eval/download.py --reference
+$env:SEARCHI_FORCE_CPU = '1'
+& tools/semantic-proof/.venv/Scripts/python.exe tools/rerank-eval/reference.py --output .searchi/rerank-eval/reference-v1.json
+```
+
+`reference-plan.json` retains five short pairs and adds a 256-token budget case.
+F32 parity passed; quantized score differences persisted. Mixed-length padded
+batching was slower. Neither result establishes general absence calibration or
+sufficient default-search latency. Detailed costs and possible VM contention are
+recorded in the linked report documentation.

tools/rerank-eval/compatibility-plan.json

@@ -0,0 +1,38 @@
+{
+  "version": 1,
+  "status": "pre-inference frozen compatibility/cost contract, not relevance calibration",
+  "cpu_provider": "CPUExecutionProvider",
+  "intra_op_threads": 1,
+  "inter_op_threads": 1,
+  "parallel_execution": false,
+  "max_pairs": 5,
+  "max_pair_tokens": 256,
+  "max_query_tokens": 64,
+  "warm_iterations": 20,
+  "parity_absolute_tolerance": 0.001,
+  "parity_relative_tolerance": 0.001,
+  "added_warm_p95_target_ms": 200,
+  "no_suppression": true,
+  "pairs": [
+    [
+      "How do we cancel the supplier partnership?",
+      "Either party may terminate the supplier contract by giving sixty days written notice."
+    ],
+    [
+      "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
+      "Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen."
+    ],
+    [
+      "How do we cancel the supplier partnership?",
+      "Tomatoes require sunshine and consistent watering."
+    ],
+    [
+      "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
+      "Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung."
+    ],
+    [
+      "Where does the code implement retry backoff?",
+      "fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }"
+    ]
+  ]
+}

tools/rerank-eval/compatibility.py

@@ -0,0 +1,131 @@
+"""Owned CPU-only cross-encoder compatibility/cost diagnostic; never alters search."""
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import platform
+import statistics
+import tempfile
+import time
+
+import numpy as np
+import onnxruntime as ort
+from tokenizers import Tokenizer
+from download import DESTINATION, digest
+
+EXPECTED_PLAN = "02569e771ab74ff593842ff26d7eb42d1b0509bef09a1867cf0f1be6b1dc5a5a"
+EXPECTED_MANIFEST = "92ec98e8ccb5f8e8e08fbf46c7a4eb1407479b4dd0fafaabf6dc7cfc7aae2da7"
+
+
+def checked_json(path, expected):
+    data = path.read_bytes()
+    if hashlib.sha256(data).hexdigest() != expected:
+        raise ValueError("Frozen diagnostic inputs changed")
+    return json.loads(data)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    owned_root = DESTINATION.parents[1].resolve()
+    output = args.output.resolve()
+    if not output.is_relative_to(owned_root) or output.exists():
+        raise ValueError("Output must be new and under owned .searchi data")
+    if os.environ.get("SEARCHI_FORCE_CPU") != "1":
+        raise ValueError("Windows VM diagnostic requires explicit SEARCHI_FORCE_CPU=1")
+    plan = checked_json(Path(__file__).with_name("compatibility-plan.json"), EXPECTED_PLAN)
+    manifest = checked_json(Path(__file__).with_name("model-manifest.json"), EXPECTED_MANIFEST)
+    start = time.perf_counter()
+    for item in manifest["files"]:
+        path = DESTINATION / item["path"]
+        if path.stat().st_size != item["bytes"] or digest(path) != item["sha256"]:
+            raise ValueError("Pinned diagnostic asset mismatch")
+    verification_ms = (time.perf_counter() - start) * 1000
+    start = time.perf_counter()
+    tokenizer = Tokenizer.from_file(str(DESTINATION / "tokenizer.json"))
+    special = tokenizer.num_special_tokens_to_add(True)
+    encodings = []
+    for query, passage in plan["pairs"]:
+        query_encoding = tokenizer.encode(query, add_special_tokens=False)
+        query_encoding.truncate(plan["max_query_tokens"])
+        passage_encoding = tokenizer.encode(passage, add_special_tokens=False)
+        passage_encoding.truncate(plan["max_pair_tokens"] - len(query_encoding.ids) - special)
+        encoding = tokenizer.post_process(query_encoding, passage_encoding, add_special_tokens=True)
+        if len(encoding.ids) > plan["max_pair_tokens"]:
+            raise ValueError("Pair token budget exceeded")
+        encodings.append(encoding)
+    tokenization_ms = (time.perf_counter() - start) * 1000
+    options = ort.SessionOptions()
+    options.intra_op_num_threads = 1
+    options.inter_op_num_threads = 1
+    options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+    start = time.perf_counter()
+    session = ort.InferenceSession(str(DESTINATION / "onnx/model_quint8_avx2.onnx"), sess_options=options, providers=["CPUExecutionProvider"])
+    startup_ms = (time.perf_counter() - start) * 1000
+    if session.get_providers() != ["CPUExecutionProvider"]:
+        raise ValueError("Unexpected execution provider")
+    names = [item.name for item in session.get_inputs()]
+    if not set(names) <= {"input_ids", "attention_mask", "token_type_ids"}:
+        raise ValueError("Unexpected model input contract")
+    def score(items):
+        width = max(len(item.ids) for item in items)
+        ids = np.full((len(items), width), 1, dtype=np.int64)
+        mask = np.zeros_like(ids)
+        types = np.zeros_like(ids)
+        for index, item in enumerate(items):
+            count = len(item.ids)
+            ids[index, :count] = item.ids
+            mask[index, :count] = item.attention_mask
+            types[index, :count] = item.type_ids
+        values = {"input_ids": ids, "attention_mask": mask, "token_type_ids": types}
+        logits = session.run(None, {name: values[name] for name in names})[0]
+        if logits.shape != (len(items), 1) or not np.isfinite(logits).all():
+            raise ValueError("Invalid classifier output")
+        return logits[:, 0]
+    start = time.perf_counter()
+    serial = np.asarray([float(score([item])[0]) for item in encodings])
+    first_serial_ms = (time.perf_counter() - start) * 1000
+    start = time.perf_counter()
+    batch = score(encodings)
+    first_batch_ms = (time.perf_counter() - start) * 1000
+    parity = bool(np.allclose(serial, batch, atol=plan["parity_absolute_tolerance"], rtol=plan["parity_relative_tolerance"]))
+    serial_timings = []
+    batch_timings = []
+    for _ in range(plan["warm_iterations"]):
+        start = time.perf_counter()
+        for item in encodings:
+            score([item])
+        serial_timings.append((time.perf_counter() - start) * 1000)
+        start = time.perf_counter()
+        score(encodings)
+        batch_timings.append((time.perf_counter() - start) * 1000)
+    def timing(values):
+        return {"median_ms": statistics.median(values), "p95_ms": sorted(values)[math.ceil(len(values) * .95) - 1], "samples_ms": values}
+    report = {"complete": True, "plan_sha256": EXPECTED_PLAN, "manifest_sha256": EXPECTED_MANIFEST,
+        "platform": platform.platform(), "python": platform.python_version(), "onnxruntime": ort.__version__,
+        "numpy": np.__version__, "providers": session.get_providers(), "cpu_threads": 1,
+        "asset_verification_ms": verification_ms, "tokenization_ms": tokenization_ms, "session_startup_ms": startup_ms,
+        "pair_tokens": [len(item.ids) for item in encodings], "first_serial_ms": first_serial_ms, "first_batch_ms": first_batch_ms,
+        "serial_logits": serial.tolist(), "batch_logits": batch.tolist(), "parity_passed": parity,
+        "max_absolute_difference": float(np.max(np.abs(serial - batch))),
+        "warm_serial": timing(serial_timings), "warm_batch": timing(batch_timings),
+        "limitations": "Five synthetic short pairs; classifier compatibility/cost only. No search pipeline or absence calibration. No confidence probability or suppression."}
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output.parent, delete=False) as handle:
+        temporary = Path(handle.name)
+        json.dump(report, handle, indent=2)
+        handle.write("\n")
+    try:
+        os.link(temporary, output)
+    finally:
+        temporary.unlink(missing_ok=True)
+    print(json.dumps({"output": str(output), "parity_passed": parity, "max_absolute_difference": report["max_absolute_difference"], "warm_serial": report["warm_serial"]["p95_ms"], "warm_batch": report["warm_batch"]["p95_ms"]}), flush=True)
+    if not parity:
+        raise SystemExit("Serial/batch parity failed; do not deploy")
+
+
+if __name__ == "__main__":
+    main()

tools/rerank-eval/download.py

@@ -0,0 +1,53 @@
+"""Download only pinned diagnostic assets into owned ignored data."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import time
+import urllib.request
+import uuid
+
+ROOT = Path(__file__).resolve().parents[2]
+DESTINATION = ROOT / ".searchi" / "rerank-eval" / "model"
+
+
+def digest(path):
+    checksum = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(65536), b""):
+            checksum.update(chunk)
+    return checksum.hexdigest()
+
+
+def main():
+    parser=argparse.ArgumentParser()
+    parser.add_argument("--reference",action="store_true")
+    args=parser.parse_args()
+    name="reference-manifest.json" if args.reference else "model-manifest.json"
+    manifest = json.loads(Path(__file__).with_name(name).read_text(encoding="utf-8"))
+    for item in manifest["files"]:
+        destination = DESTINATION / item["path"]
+        if destination.is_file() and destination.stat().st_size == item["bytes"] and digest(destination) == item["sha256"]:
+            print("verified cache", item["path"], flush=True)
+            continue
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        temporary = destination.with_name(destination.name + "." + uuid.uuid4().hex + ".partial")
+        total = 0
+        start = time.monotonic()
+        try:
+            with urllib.request.urlopen(item["url"], timeout=20) as response, temporary.open("xb") as output:
+                while chunk := response.read(65536):
+                    total += len(chunk)
+                    if total > item["bytes"] or time.monotonic() - start > 180:
+                        raise ValueError("Download exceeded size/deadline")
+                    output.write(chunk)
+            if total != item["bytes"] or digest(temporary) != item["sha256"]:
+                raise ValueError("Pinned asset hash/size mismatch")
+            temporary.replace(destination)
+            print("verified download", item["path"], total, flush=True)
+        finally:
+            temporary.unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    main()

tools/rerank-eval/model-manifest.json

@@ -0,0 +1,50 @@
+{
+  "model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
+  "revision": "1427fd652930e4ba29e8149678df786c240d8825",
+  "purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval",
+  "license": "Apache-2.0 per pinned publisher card; text from Apache",
+  "files": [
+    {
+      "path": "onnx/model_quint8_avx2.onnx",
+      "bytes": 118620016,
+      "sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx"
+    },
+    {
+      "path": "tokenizer.json",
+      "bytes": 17082660,
+      "sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json"
+    },
+    {
+      "path": "config.json",
+      "bytes": 891,
+      "sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json"
+    },
+    {
+      "path": "tokenizer_config.json",
+      "bytes": 435,
+      "sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json"
+    },
+    {
+      "path": "special_tokens_map.json",
+      "bytes": 239,
+      "sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json"
+    },
+    {
+      "path": "README.md",
+      "bytes": 2278,
+      "sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md"
+    },
+    {
+      "path": "LICENSE-APACHE-2.0.txt",
+      "bytes": 11358,
+      "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
+      "url": "https://www.apache.org/licenses/LICENSE-2.0.txt"
+    }
+  ]
+}

tools/rerank-eval/reference-manifest.json

@@ -0,0 +1,56 @@
+{
+  "model_id": "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
+  "revision": "1427fd652930e4ba29e8149678df786c240d8825",
+  "purpose": "diagnostic text reranker; Google EmbeddingGemma2 remains retrieval",
+  "license": "Apache-2.0 per pinned publisher card; text from Apache",
+  "files": [
+    {
+      "path": "onnx/model_quint8_avx2.onnx",
+      "bytes": 118620016,
+      "sha256": "6c2513767fb63d008a4377bef7a7a3555433d9436342bb53e35a3a72ffc52d4b",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model_quint8_avx2.onnx"
+    },
+    {
+      "path": "tokenizer.json",
+      "bytes": 17082660,
+      "sha256": "62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer.json"
+    },
+    {
+      "path": "config.json",
+      "bytes": 891,
+      "sha256": "cc2cfe51aa3fd759d21d21acf5dfd6994aa67a3c9210636d22e143699d336c77",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/config.json"
+    },
+    {
+      "path": "tokenizer_config.json",
+      "bytes": 435,
+      "sha256": "e7fbfbfa6347b4e414c1cee50d142e2c2f9a895dad68b068ae83a8b564c3837e",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/tokenizer_config.json"
+    },
+    {
+      "path": "special_tokens_map.json",
+      "bytes": 239,
+      "sha256": "378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/special_tokens_map.json"
+    },
+    {
+      "path": "README.md",
+      "bytes": 2278,
+      "sha256": "474736a65d6393a060119a8dc304563af67af4d8d86ccfee4a05dd0df107fc11",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/README.md"
+    },
+    {
+      "path": "LICENSE-APACHE-2.0.txt",
+      "bytes": 11358,
+      "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
+      "url": "https://www.apache.org/licenses/LICENSE-2.0.txt"
+    },
+    {
+      "path": "onnx/model.onnx",
+      "bytes": 470883696,
+      "sha256": "3e9a03ed1e966f7c5288dd4230e3d6a9bf5e3a170a06f1f4241c5bca12c6487c",
+      "url": "https://huggingface.co/cross-encoder/mmarco-mMiniLMv2-L12-H384-v1/resolve/1427fd652930e4ba29e8149678df786c240d8825/onnx/model.onnx"
+    }
+  ]
+}

tools/rerank-eval/reference-plan.json

@@ -0,0 +1,44 @@
+{
+  "version": 2,
+  "status": "pre-inference F32 reference parity and 256-token cost contract; original INT8 failure unchanged",
+  "cpu_provider": "CPUExecutionProvider",
+  "intra_op_threads": 1,
+  "inter_op_threads": 1,
+  "parallel_execution": false,
+  "max_pairs": 6,
+  "max_pair_tokens": 256,
+  "max_query_tokens": 64,
+  "warm_iterations": 20,
+  "parity_absolute_tolerance": 0.001,
+  "parity_relative_tolerance": 0.001,
+  "added_warm_p95_target_ms": 200,
+  "no_suppression": true,
+  "pairs": [
+    [
+      "How do we cancel the supplier partnership?",
+      "Either party may terminate the supplier contract by giving sixty days written notice."
+    ],
+    [
+      "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
+      "Jede Partei kann die Zusammenarbeit mit dem Lieferanten mit einer Frist von sechzig Tagen kündigen."
+    ],
+    [
+      "How do we cancel the supplier partnership?",
+      "Tomatoes require sunshine and consistent watering."
+    ],
+    [
+      "Wie beenden wir die Zusammenarbeit mit dem Lieferanten?",
+      "Der Garten benötigt Sonnenlicht und regelmäßige Bewässerung."
+    ],
+    [
+      "Where does the code implement retry backoff?",
+      "fn retry_delay(attempt: u32) -> u64 { 100 * (1 << attempt.min(6)) }"
+    ],
+    [
+      "supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice supplier partnership cancellation notice ",
+      "Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplier partnership with sixty days written notice. Either party may terminate the supplLine truncated
+    ]
+  ],
+  "reference_model": "onnx/model.onnx",
+  "comparison_model": "onnx/model_quint8_avx2.onnx"
+}

tools/rerank-eval/reference.py

@@ -0,0 +1,143 @@
+"""Owned CPU-only cross-encoder compatibility/cost diagnostic; never alters search."""
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import platform
+import statistics
+import tempfile
+import time
+
+import numpy as np
+import onnxruntime as ort
+from tokenizers import Tokenizer
+from download import DESTINATION, digest
+
+EXPECTED_PLAN = "174867d4212b222e5302b1b16832b371c6bd38ec991f4145c952ed35446341f1"
+EXPECTED_MANIFEST = "3c252dfc10e03ebba59a7ff0e33e6e2183a26c71862c52f0700f27ba659142be"
+
+
+def checked_json(path, expected):
+    data = path.read_bytes()
+    if hashlib.sha256(data).hexdigest() != expected:
+        raise ValueError("Frozen diagnostic inputs changed")
+    return json.loads(data)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    owned_root = DESTINATION.parents[1].resolve()
+    output = args.output.resolve()
+    if not output.is_relative_to(owned_root) or output.exists():
+        raise ValueError("Output must be new and under owned .searchi data")
+    if os.environ.get("SEARCHI_FORCE_CPU") != "1":
+        raise ValueError("Windows VM diagnostic requires explicit SEARCHI_FORCE_CPU=1")
+    plan = checked_json(Path(__file__).with_name("reference-plan.json"), EXPECTED_PLAN)
+    manifest = checked_json(Path(__file__).with_name("reference-manifest.json"), EXPECTED_MANIFEST)
+    start = time.perf_counter()
+    for item in manifest["files"]:
+        path = DESTINATION / item["path"]
+        if path.stat().st_size != item["bytes"] or digest(path) != item["sha256"]:
+            raise ValueError("Pinned diagnostic asset mismatch")
+    verification_ms = (time.perf_counter() - start) * 1000
+    start = time.perf_counter()
+    tokenizer = Tokenizer.from_file(str(DESTINATION / "tokenizer.json"))
+    special = tokenizer.num_special_tokens_to_add(True)
+    encodings = []
+    for query, passage in plan["pairs"]:
+        query_encoding = tokenizer.encode(query, add_special_tokens=False)
+        query_encoding.truncate(plan["max_query_tokens"])
+        passage_encoding = tokenizer.encode(passage, add_special_tokens=False)
+        passage_encoding.truncate(plan["max_pair_tokens"] - len(query_encoding.ids) - special)
+        encoding = tokenizer.post_process(query_encoding, passage_encoding, add_special_tokens=True)
+        if len(encoding.ids) > plan["max_pair_tokens"]:
+            raise ValueError("Pair token budget exceeded")
+        encodings.append(encoding)
+    tokenization_ms = (time.perf_counter() - start) * 1000
+    options = ort.SessionOptions()
+    options.intra_op_num_threads = 1
+    options.inter_op_num_threads = 1
+    options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+    start = time.perf_counter()
+    session = ort.InferenceSession(str(DESTINATION / plan["reference_model"]), sess_options=options, providers=["CPUExecutionProvider"])
+    startup_ms = (time.perf_counter() - start) * 1000
+    if session.get_providers() != ["CPUExecutionProvider"]:
+        raise ValueError("Unexpected execution provider")
+    names = [item.name for item in session.get_inputs()]
+    if not set(names) <= {"input_ids", "attention_mask", "token_type_ids"}:
+        raise ValueError("Unexpected model input contract")
+    def score(items):
+        width = max(len(item.ids) for item in items)
+        ids = np.full((len(items), width), 1, dtype=np.int64)
+        mask = np.zeros_like(ids)
+        types = np.zeros_like(ids)
+        for index, item in enumerate(items):
+            count = len(item.ids)
+            ids[index, :count] = item.ids
+            mask[index, :count] = item.attention_mask
+            types[index, :count] = item.type_ids
+        values = {"input_ids": ids, "attention_mask": mask, "token_type_ids": types}
+        logits = session.run(None, {name: values[name] for name in names})[0]
+        if logits.shape != (len(items), 1) or not np.isfinite(logits).all():
+            raise ValueError("Invalid classifier output")
+        return logits[:, 0]
+    start = time.perf_counter()
+    serial = np.asarray([float(score([item])[0]) for item in encodings])
+    first_serial_ms = (time.perf_counter() - start) * 1000
+    start = time.perf_counter()
+    batch = score(encodings)
+    first_batch_ms = (time.perf_counter() - start) * 1000
+    parity = bool(np.allclose(serial, batch, atol=plan["parity_absolute_tolerance"], rtol=plan["parity_relative_tolerance"]))
+    serial_timings = []
+    batch_timings = []
+    for _ in range(plan["warm_iterations"]):
+        start = time.perf_counter()
+        for item in encodings:
+            score([item])
+        serial_timings.append((time.perf_counter() - start) * 1000)
+        start = time.perf_counter()
+        score(encodings)
+        batch_timings.append((time.perf_counter() - start) * 1000)
+    reference_session = session
+    session = ort.InferenceSession(str(DESTINATION / plan["comparison_model"]), sess_options=options, providers=["CPUExecutionProvider"])
+    if session.get_providers() != ["CPUExecutionProvider"]:
+        raise ValueError("Unexpected comparison provider")
+    quantized_serial = np.asarray([float(score([item])[0]) for item in encodings])
+    quantized_batch = score(encodings)
+    session = reference_session
+    def timing(values):
+        return {"median_ms": statistics.median(values), "p95_ms": sorted(values)[math.ceil(len(values) * .95) - 1], "samples_ms": values}
+    report = {"complete": True, "plan_sha256": EXPECTED_PLAN, "manifest_sha256": EXPECTED_MANIFEST,
+        "platform": platform.platform(), "python": platform.python_version(), "onnxruntime": ort.__version__,
+        "numpy": np.__version__, "providers": session.get_providers(), "cpu_threads": 1,
+        "asset_verification_ms": verification_ms, "tokenization_ms": tokenization_ms, "session_startup_ms": startup_ms,
+        "pair_tokens": [len(item.ids) for item in encodings], "first_serial_ms": first_serial_ms, "first_batch_ms": first_batch_ms,
+        "reference_model": plan["reference_model"], "comparison_model": plan["comparison_model"],
+        "serial_logits": serial.tolist(), "batch_logits": batch.tolist(), "parity_passed": parity,
+        "quantized_serial_logits": quantized_serial.tolist(), "quantized_batch_logits": quantized_batch.tolist(),
+        "quantized_vs_reference_serial_max_abs": float(np.max(np.abs(serial-quantized_serial))),
+        "quantized_batch_max_abs": float(np.max(np.abs(quantized_serial-quantized_batch))),
+        "five_short_pairs_ordering_agrees": bool(np.array_equal(np.argsort(serial[:5]),np.argsort(quantized_serial[:5]))),
+        "max_absolute_difference": float(np.max(np.abs(serial - batch))),
+        "warm_serial": timing(serial_timings), "warm_batch": timing(batch_timings),
+        "limitations": "Five synthetic short pairs plus one 256-token cost pair; classifier compatibility/cost only. No search pipeline or absence calibration. No confidence probability or suppression."}
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output.parent, delete=False) as handle:
+        temporary = Path(handle.name)
+        json.dump(report, handle, indent=2)
+        handle.write("\n")
+    try:
+        os.link(temporary, output)
+    finally:
+        temporary.unlink(missing_ok=True)
+    print(json.dumps({"output": str(output), "parity_passed": parity, "max_absolute_difference": report["max_absolute_difference"], "warm_serial": report["warm_serial"]["p95_ms"], "warm_batch": report["warm_batch"]["p95_ms"]}), flush=True)
+    if not parity:
+        raise SystemExit("Serial/batch parity failed; do not deploy")
+
+
+if __name__ == "__main__":
+    main()