diff --git a/docs/RELEVANCE-CALIBRATION.md b/docs/RELEVANCE-CALIBRATION.md index 31b836b..36cea12 100644 --- a/docs/RELEVANCE-CALIBRATION.md +++ b/docs/RELEVANCE-CALIBRATION.md @@ -108,3 +108,43 @@ Policy SHA-256: `df0a887b97dd7fda6aeff1b6c966fe7558521f40dbe1c911e91cfd312dca7461`. Ignored analysis: `.searchi/qa/calibration-cosine-policy-v1.json`. Held-out remains unevaluated at this freeze. Product ranking is unchanged. + +## Held-out result and disposition, 2026-10-13 + +After the policy freeze was committed and pushed, actual Google CPU inference +completed all 24 held-out queries plus six Code-scope repeats. The fixture, +model, policy and threshold were unchanged. The explicit runner option requires +the exact pre-evaluation policy SHA; default execution remains calibration-only. +Four structural Rust tests and strict all-target Clippy passed. + +| Scope | Unfiltered semantic / hybrid positive top-1 | Positive top results retained at 0.70 | Absent requests with retained semantic neighbours | +| --- | --- | --- | --- | +| All files | 16/16 | 15/16 | 0/8 | +| Additional Code scope | 4/4 | 4/4 | 0/2 | + +The lost request is `version-wildcard-de`: its correct All files semantic +candidate scores **0.6766**, below the frozen threshold. It has no lexical +results, so preserving lexical retrieval would not rescue it. The same request +under Code scope scores **0.7388** and survives. The highest absent All files +top score is approximately **0.6954**, exceeding that positive score: there is +no single raw-cosine threshold separating these held-out top results. + +**Reject deployment as a universal default cutoff.** Do not lower this policy +after seeing held-out outcomes or claim that its strong calibration result +proved safe abstention. Product ranking remains unchanged. The next relevance +work should preserve access to weak-but-correct results and evaluate richer +evidence on a new independent corpus; this sample is no longer unseen data. + +`scores.hypothesis_semantic` measures retained semantic candidates only. +The recorded production hybrid request is unmodified; no filtered hybrid +fusion or UI confidence behavior was executed. This small bilingual corpus +does not establish general customer quality or media abstention. + +Raw ignored report: `.searchi/qa/held-out-cosine-v1.json`, SHA-256 +`8f6baceac27e90c719c2bbb522eeb2d1623056e2f8334f31f7dd67a912c0fccd`. +Diagnostic runner SHA-256: +`db685085f7f53d02cc85ef7c534e8ac02f4c5c806940bd4ff44ccbeae76e5762`. +Worker capabilities record Google EmbeddingGemma 2, CPU, one thread, query +role, context 512 and 768 dimensions. Semantic indexing took 73,991.6605 ms; +query startup took 7,153.4587 ms. These ordered debug setup measurements are +not UI latency or warm p95. Owned workers exited; the installed app was retained. diff --git a/tools/relevance-eval/README.md b/tools/relevance-eval/README.md index 8b6068e..cf28b2d 100644 --- a/tools/relevance-eval/README.md +++ b/tools/relevance-eval/README.md @@ -1,10 +1,11 @@ -# Calibration-only relevance diagnostic +# Frozen relevance diagnostic This internal runner uses searchi's shared Rust engine and genuine Google -EmbeddingGemma 2. It has **no held-out execution option**. It indexes all 14 frozen -source files as the fixed distractor library, but evaluates only the 24 -calibration queries. Held-out queries stay unevaluated until a policy and its -parameters are frozen using calibration evidence. +EmbeddingGemma 2. By default it indexes all 14 frozen source files as the fixed +distractor library and evaluates only the 24 calibration queries. Held-out +execution requires `--held-out-policy` with the exact SHA-256 of the hypothesis +frozen in commit `e7245fe`, before held-out inference. Changed policy bytes are +rejected. The two splits are selected independently and never mixed in one run. First verify all frozen labels, source bytes, licenses and split integrity: @@ -21,6 +22,14 @@ $env:SEARCHI_FORCE_CPU = '1' tools/relevance-eval/target/release/searchi-relevance-eval.exe fixtures/relevance-v2 C:/path/to/verified-models C:/path/to/searchi-gemma-worker.exe .searchi/qa/calibration-v2.json ``` +For the one frozen held-out evaluation, append +`--held-out-policy tools/relevance-eval/policies/cosine-v1.json` and use a new +held-out report name. Raw lexical, semantic and production hybrid results are +retained. `scores.hypothesis_semantic` evaluates raw semantic candidates after +the frozen cutoff; it is not a newly executed filtered hybrid fusion. This +diagnostic does not deploy a threshold into the shared engine. Do not retune +this policy using the held-out outcomes. + Supply an existing output directory and a new output filename. Existing evidence is never overwritten: the output is reserved with `create_new` before costly work and subsequent phase/row checkpoints replace JSON atomically. The runner verifies fixture SHA-256, exact source bytes, diff --git a/tools/relevance-eval/src/main.rs b/tools/relevance-eval/src/main.rs index 3501d61..f20c87e 100644 --- a/tools/relevance-eval/src/main.rs +++ b/tools/relevance-eval/src/main.rs @@ -1,4 +1,4 @@ -//! Calibration-only diagnostics through the production shared engine. +//! Frozen calibration and policy-gated held-out diagnostics through the shared engine. use anyhow::{ensure, Context, Result}; use searchi_core::{SearchIndex, SearchOptions, SearchResult}; use searchi_service::HybridSearch; @@ -26,6 +26,14 @@ fn digest(path: &Path) -> Result { } fn calibration_queries(fixture: &Value) -> Result> { + split_queries(fixture, "calibration") +} + +fn split_queries<'a>(fixture: &'a Value, split: &str) -> Result> { + ensure!( + matches!(split, "calibration" | "held_out"), + "Unknown requested split" + ); let queries = fixture["queries"].as_array().context("Missing queries")?; ensure!( queries @@ -33,13 +41,10 @@ fn calibration_queries(fixture: &Value) -> Result> { .all(|q| matches!(q["split"].as_str(), Some("calibration" | "held_out"))), "Unknown split" ); - let selected: Vec<_> = queries - .iter() - .filter(|q| q["split"] == "calibration") - .collect(); + let selected: Vec<_> = queries.iter().filter(|q| q["split"] == split).collect(); ensure!( selected.len() == 24, - "Expected exactly 24 frozen calibration queries" + "Expected exactly 24 frozen queries in selected split" ); Ok(selected) } @@ -56,7 +61,11 @@ fn checkpoint(path: &Path, report: &Value) -> Result<()> { Ok(()) } -fn score(hits: &[SearchResult], query: &Value, fixture: &Value) -> Result { +fn score<'a>( + hits: impl IntoIterator, + query: &Value, + fixture: &Value, +) -> Result { let expected = query["expected_ids"] .as_array() .context("Missing expected labels")?; @@ -89,8 +98,8 @@ fn score(hits: &[SearchResult], query: &Value, fixture: &Value) -> Result fn main() -> Result<()> { let args: Vec<_> = std::env::args_os().skip(1).collect(); ensure!( - args.len() == 4, - "Supply FIXTURE MODEL_DIR WORKER OUTPUT_JSON; held-out execution is unsupported" + args.len() == 4 || (args.len() == 6 && args[4] == "--held-out-policy"), + "Supply FIXTURE MODEL_DIR WORKER OUTPUT_JSON [--held-out-policy FROZEN_POLICY]" ); ensure!( std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"), @@ -100,12 +109,27 @@ fn main() -> Result<()> { let models = PathBuf::from(&args[1]).canonicalize()?; let worker = PathBuf::from(&args[2]).canonicalize()?; let output = PathBuf::from(&args[3]); + let policy = if args.len() == 6 { + let path = PathBuf::from(&args[5]); + ensure!( + digest(&path)? == "df0a887b97dd7fda6aeff1b6c966fe7558521f40dbe1c911e91cfd312dca7461", + "Policy differs from frozen pre-evaluation hypothesis" + ); + Some(serde_json::from_slice::(&fs::read(path)?)?) + } else { + None + }; + let split = if policy.is_some() { + "held_out" + } else { + "calibration" + }; fs::OpenOptions::new() .write(true) .create_new(true) .open(&output) .context("Cannot reserve output; existing evidence is never overwritten")?; - let mut report = json!({"complete":false,"phase":"verifying","split":"calibration","held_out_evaluated":false,"cpu_threads":1,"rows":[]}); + let mut report = json!({"complete":false,"phase":"verifying","split":split,"held_out_evaluated":policy.is_some(),"policy":policy,"cpu_threads":1,"rows":[]}); checkpoint(&output, &report)?; let fixture_path = fixture_root.join("fixture.json"); let fixture: Value = serde_json::from_slice(&fs::read(&fixture_path)?)?; @@ -119,7 +143,11 @@ fn main() -> Result<()> { lock["sha256"]["fixture.json"] == digest(&fixture_path)?, "Frozen fixture changed" ); - let queries = calibration_queries(&fixture)?; + let queries = if policy.is_some() { + split_queries(&fixture, split)? + } else { + calibration_queries(&fixture)? + }; let manifest_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json"); let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?; @@ -145,7 +173,7 @@ fn main() -> Result<()> { let temporary = tempfile::tempdir()?; let corpus = temporary.path().join("corpus"); fs::create_dir(&corpus)?; - // Index both splits as the fixed distractor library, but execute calibration queries only. + // Index both splits as the fixed distractor library, execute only the selected split. for asset in fixture["assets"].as_array().context("Missing assets")? { let source = fixture_root .join(asset["path"].as_str().context("Missing asset path")?) @@ -189,18 +217,19 @@ fn main() -> Result<()> { let query_startup_ms = start.elapsed().as_secs_f64() * 1000.0; report["worker_info"] = json!(query_session.worker_info()); report["query_startup_ms"] = json!(query_startup_ms); - report["phase"] = json!("calibration_queries"); + report["phase"] = json!(format!("{split}_queries")); checkpoint(&output, &report)?; for query in queries { - ensure!( - query["split"] == "calibration", - "Held-out execution forbidden" - ); + ensure!(query["split"] == split, "Query outside selected split"); let text = query["query"].as_str().context("Missing query text")?; let code_intent = query["id"] .as_str() .context("Missing query ID")? - .starts_with("pattern-"); + .starts_with(if split == "held_out" { + "version-" + } else { + "pattern-" + }); for code_only in [false, true] .into_iter() .filter(|code_only| !*code_only || code_intent) @@ -215,7 +244,15 @@ fn main() -> Result<()> { let semantic_hits = query_session.search_semantic(text, 80, &options)?; let semantic_ms = start.elapsed().as_secs_f64() * 1000.0; let hybrid_hits = query_session.search_measured(text, 20, &options, true)?; - let scores = json!({"lexical":score(&lexical_hits.results,query,&fixture)?,"semantic":score(&semantic_hits,query,&fixture)?,"hybrid":score(&hybrid_hits.results,query,&fixture)?}); + let retained: Vec<_> = semantic_hits + .iter() + .filter(|hit| { + policy + .as_ref() + .is_none_or(|p| hit.score >= p["semantic_cosine_minimum"].as_f64().unwrap()) + }) + .collect(); + let scores = json!({"lexical":score(&lexical_hits.results,query,&fixture)?,"semantic":score(&semantic_hits,query,&fixture)?,"hybrid":score(&hybrid_hits.results,query,&fixture)?,"hypothesis_semantic":score(retained,query,&fixture)?}); report["rows"].as_array_mut().context("Missing output rows")?.push(json!({"query":query,"scope":if code_only {"code"} else {"all_files"},"lexical":lexical_hits,"semantic_raw":semantic_hits,"semantic_ms":semantic_ms,"hybrid":hybrid_hits,"scores":scores,"order_note":"lexical then semantic then hybrid; hybrid reuses the preceding query-vector cache. Raw semantic scores are cosine, hybrid scores are RRF, not confidence."})); checkpoint(&output, &report)?; } @@ -262,6 +299,24 @@ mod tests { assert!(calibration_queries(&json!({"queries":[{"split":"held_out"}]})).is_err()); } + #[test] + fn held_out_selection_is_disjoint_and_has_six_code_requests() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../fixtures/relevance-v2/fixture.json"); + let fixture: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let selected = split_queries(&fixture, "held_out").unwrap(); + assert_eq!(selected.len(), 24); + assert!(selected.iter().all(|q| q["split"] == "held_out")); + assert_eq!( + selected + .iter() + .filter(|q| q["id"].as_str().unwrap().starts_with("version-")) + .count(), + 6 + ); + assert!(split_queries(&fixture, "unknown").is_err()); + } + #[test] fn frozen_calibration_has_six_code_requests_and_disjoint_query_ids() { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))