karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Evaluate frozen relevance hypothesis once on held-out Google CPU queries

Browse Source
This commit is contained in:
karl committed 2026-10-13 12:23:31 +02:00
1 parent e7245fe15e
commit 193ab08f62
3 files changed
+128 -24
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/relevance-eval/README.md
+14 -5
View File
@@ -1,10 +1,11 @@
# Calibration-only relevance diagnostic
# Frozen relevance diagnostic
This internal runner uses searchi's shared Rust engine and genuine Google
EmbeddingGemma 2. It has **no held-out execution option**. It indexes all 14 frozen
source files as the fixed distractor library, but evaluates only the 24
calibration queries. Held-out queries stay unevaluated until a policy and its
parameters are frozen using calibration evidence.
EmbeddingGemma 2. By default it indexes all 14 frozen source files as the fixed
distractor library and evaluates only the 24 calibration queries. Held-out
execution requires `--held-out-policy` with the exact SHA-256 of the hypothesis
frozen in commit `e7245fe`, before held-out inference. Changed policy bytes are
rejected. The two splits are selected independently and never mixed in one run.
First verify all frozen labels, source bytes, licenses and split integrity:
@@ -21,6 +22,14 @@ $env:SEARCHI_FORCE_CPU = '1'
tools/relevance-eval/target/release/searchi-relevance-eval.exe fixtures/relevance-v2 C:/path/to/verified-models C:/path/to/searchi-gemma-worker.exe .searchi/qa/calibration-v2.json
```
For the one frozen held-out evaluation, append
`--held-out-policy tools/relevance-eval/policies/cosine-v1.json` and use a new
held-out report name. Raw lexical, semantic and production hybrid results are
retained. `scores.hypothesis_semantic` evaluates raw semantic candidates after
the frozen cutoff; it is not a newly executed filtered hybrid fusion. This
diagnostic does not deploy a threshold into the shared engine. Do not retune
this policy using the held-out outcomes.
Supply an existing output directory and a new output filename. Existing evidence
is never overwritten: the output is reserved with `create_new` before costly
work and subsequent phase/row checkpoints replace JSON atomically. The runner verifies fixture SHA-256, exact source bytes,
tools/relevance-eval/src/main.rs
+74 -19
View File
@@ -1,4 +1,4 @@
//! Calibration-only diagnostics through the production shared engine.
//! Frozen calibration and policy-gated held-out diagnostics through the shared engine.
use anyhow::{ensure, Context, Result};
use searchi_core::{SearchIndex, SearchOptions, SearchResult};
use searchi_service::HybridSearch;
@@ -26,6 +26,14 @@ fn digest(path: &Path) -> Result<String> {
}
fn calibration_queries(fixture: &Value) -> Result<Vec<&Value>> {
split_queries(fixture, "calibration")
}
fn split_queries<'a>(fixture: &'a Value, split: &str) -> Result<Vec<&'a Value>> {
ensure!(
matches!(split, "calibration" | "held_out"),
"Unknown requested split"
);
let queries = fixture["queries"].as_array().context("Missing queries")?;
ensure!(
queries
@@ -33,13 +41,10 @@ fn calibration_queries(fixture: &Value) -> Result<Vec<&Value>> {
.all(|q| matches!(q["split"].as_str(), Some("calibration" | "held_out"))),
"Unknown split"
);
let selected: Vec<_> = queries
.iter()
.filter(|q| q["split"] == "calibration")
.collect();
let selected: Vec<_> = queries.iter().filter(|q| q["split"] == split).collect();
ensure!(
selected.len() == 24,
"Expected exactly 24 frozen calibration queries"
"Expected exactly 24 frozen queries in selected split"
);
Ok(selected)
}
@@ -56,7 +61,11 @@ fn checkpoint(path: &Path, report: &Value) -> Result<()> {
Ok(())
}
fn score(hits: &[SearchResult], query: &Value, fixture: &Value) -> Result<Value> {
fn score<'a>(
hits: impl IntoIterator<Item = &'a SearchResult>,
query: &Value,
fixture: &Value,
) -> Result<Value> {
let expected = query["expected_ids"]
.as_array()
.context("Missing expected labels")?;
@@ -89,8 +98,8 @@ fn score(hits: &[SearchResult], query: &Value, fixture: &Value) -> Result<Value>
fn main() -> Result<()> {
let args: Vec<_> = std::env::args_os().skip(1).collect();
ensure!(
args.len() == 4,
"Supply FIXTURE MODEL_DIR WORKER OUTPUT_JSON; held-out execution is unsupported"
args.len() == 4 || (args.len() == 6 && args[4] == "--held-out-policy"),
"Supply FIXTURE MODEL_DIR WORKER OUTPUT_JSON [--held-out-policy FROZEN_POLICY]"
);
ensure!(
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
@@ -100,12 +109,27 @@ fn main() -> Result<()> {
let models = PathBuf::from(&args[1]).canonicalize()?;
let worker = PathBuf::from(&args[2]).canonicalize()?;
let output = PathBuf::from(&args[3]);
let policy = if args.len() == 6 {
let path = PathBuf::from(&args[5]);
ensure!(
digest(&path)? == "df0a887b97dd7fda6aeff1b6c966fe7558521f40dbe1c911e91cfd312dca7461",
"Policy differs from frozen pre-evaluation hypothesis"
);
Some(serde_json::from_slice::<Value>(&fs::read(path)?)?)
} else {
None
};
let split = if policy.is_some() {
"held_out"
} else {
"calibration"
};
fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&output)
.context("Cannot reserve output; existing evidence is never overwritten")?;
let mut report = json!({"complete":false,"phase":"verifying","split":"calibration","held_out_evaluated":false,"cpu_threads":1,"rows":[]});
let mut report = json!({"complete":false,"phase":"verifying","split":split,"held_out_evaluated":policy.is_some(),"policy":policy,"cpu_threads":1,"rows":[]});
checkpoint(&output, &report)?;
let fixture_path = fixture_root.join("fixture.json");
let fixture: Value = serde_json::from_slice(&fs::read(&fixture_path)?)?;
@@ -119,7 +143,11 @@ fn main() -> Result<()> {
lock["sha256"]["fixture.json"] == digest(&fixture_path)?,
"Frozen fixture changed"
);
let queries = calibration_queries(&fixture)?;
let queries = if policy.is_some() {
split_queries(&fixture, split)?
} else {
calibration_queries(&fixture)?
};
let manifest_path =
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json");
let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
@@ -145,7 +173,7 @@ fn main() -> Result<()> {
let temporary = tempfile::tempdir()?;
let corpus = temporary.path().join("corpus");
fs::create_dir(&corpus)?;
// Index both splits as the fixed distractor library, but execute calibration queries only.
// Index both splits as the fixed distractor library, execute only the selected split.
for asset in fixture["assets"].as_array().context("Missing assets")? {
let source = fixture_root
.join(asset["path"].as_str().context("Missing asset path")?)
@@ -189,18 +217,19 @@ fn main() -> Result<()> {
let query_startup_ms = start.elapsed().as_secs_f64() * 1000.0;
report["worker_info"] = json!(query_session.worker_info());
report["query_startup_ms"] = json!(query_startup_ms);
report["phase"] = json!("calibration_queries");
report["phase"] = json!(format!("{split}_queries"));
checkpoint(&output, &report)?;
for query in queries {
ensure!(
query["split"] == "calibration",
"Held-out execution forbidden"
);
ensure!(query["split"] == split, "Query outside selected split");
let text = query["query"].as_str().context("Missing query text")?;
let code_intent = query["id"]
.as_str()
.context("Missing query ID")?
.starts_with("pattern-");
.starts_with(if split == "held_out" {
"version-"
} else {
"pattern-"
});
for code_only in [false, true]
.into_iter()
.filter(|code_only| !*code_only || code_intent)
@@ -215,7 +244,15 @@ fn main() -> Result<()> {
let semantic_hits = query_session.search_semantic(text, 80, &options)?;
let semantic_ms = start.elapsed().as_secs_f64() * 1000.0;
let hybrid_hits = query_session.search_measured(text, 20, &options, true)?;
let scores = json!({"lexical":score(&lexical_hits.results,query,&fixture)?,"semantic":score(&semantic_hits,query,&fixture)?,"hybrid":score(&hybrid_hits.results,query,&fixture)?});
let retained: Vec<_> = semantic_hits
.iter()
.filter(|hit| {
policy
.as_ref()
.is_none_or(|p| hit.score >= p["semantic_cosine_minimum"].as_f64().unwrap())
})
.collect();
let scores = json!({"lexical":score(&lexical_hits.results,query,&fixture)?,"semantic":score(&semantic_hits,query,&fixture)?,"hybrid":score(&hybrid_hits.results,query,&fixture)?,"hypothesis_semantic":score(retained,query,&fixture)?});
report["rows"].as_array_mut().context("Missing output rows")?.push(json!({"query":query,"scope":if code_only {"code"} else {"all_files"},"lexical":lexical_hits,"semantic_raw":semantic_hits,"semantic_ms":semantic_ms,"hybrid":hybrid_hits,"scores":scores,"order_note":"lexical then semantic then hybrid; hybrid reuses the preceding query-vector cache. Raw semantic scores are cosine, hybrid scores are RRF, not confidence."}));
checkpoint(&output, &report)?;
}
@@ -262,6 +299,24 @@ mod tests {
assert!(calibration_queries(&json!({"queries":[{"split":"held_out"}]})).is_err());
}
#[test]
fn held_out_selection_is_disjoint_and_has_six_code_requests() {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../../fixtures/relevance-v2/fixture.json");
let fixture: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap();
let selected = split_queries(&fixture, "held_out").unwrap();
assert_eq!(selected.len(), 24);
assert!(selected.iter().all(|q| q["split"] == "held_out"));
assert_eq!(
selected
.iter()
.filter(|q| q["id"].as_str().unwrap().starts_with("version-"))
.count(),
6
);
assert!(split_queries(&fixture, "unknown").is_err());
}
#[test]
fn frozen_calibration_has_six_code_requests_and_disjoint_query_ids() {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))