karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Evaluate frozen relevance hypothesis once on held-out Google CPU queries

Browse Source
This commit is contained in:
karl committed 2026-10-13 12:23:31 +02:00
1 parent e7245fe15e
commit 193ab08f62
3 files changed
+128 -24
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/relevance-eval/src/main.rs
+74 -19
View File
@@ -1,4 +1,4 @@
//! Calibration-only diagnostics through the production shared engine.
//! Frozen calibration and policy-gated held-out diagnostics through the shared engine.
use anyhow::{ensure, Context, Result};
use searchi_core::{SearchIndex, SearchOptions, SearchResult};
use searchi_service::HybridSearch;
@@ -26,6 +26,14 @@ fn digest(path: &Path) -> Result<String> {
}
fn calibration_queries(fixture: &Value) -> Result<Vec<&Value>> {
split_queries(fixture, "calibration")
}
fn split_queries<'a>(fixture: &'a Value, split: &str) -> Result<Vec<&'a Value>> {
ensure!(
matches!(split, "calibration" | "held_out"),
"Unknown requested split"
);
let queries = fixture["queries"].as_array().context("Missing queries")?;
ensure!(
queries
@@ -33,13 +41,10 @@ fn calibration_queries(fixture: &Value) -> Result<Vec<&Value>> {
.all(|q| matches!(q["split"].as_str(), Some("calibration" | "held_out"))),
"Unknown split"
);
let selected: Vec<_> = queries
.iter()
.filter(|q| q["split"] == "calibration")
.collect();
let selected: Vec<_> = queries.iter().filter(|q| q["split"] == split).collect();
ensure!(
selected.len() == 24,
"Expected exactly 24 frozen calibration queries"
"Expected exactly 24 frozen queries in selected split"
);
Ok(selected)
}
@@ -56,7 +61,11 @@ fn checkpoint(path: &Path, report: &Value) -> Result<()> {
Ok(())
}
fn score(hits: &[SearchResult], query: &Value, fixture: &Value) -> Result<Value> {
fn score<'a>(
hits: impl IntoIterator<Item = &'a SearchResult>,
query: &Value,
fixture: &Value,
) -> Result<Value> {
let expected = query["expected_ids"]
.as_array()
.context("Missing expected labels")?;
@@ -89,8 +98,8 @@ fn score(hits: &[SearchResult], query: &Value, fixture: &Value) -> Result<Value>
fn main() -> Result<()> {
let args: Vec<_> = std::env::args_os().skip(1).collect();
ensure!(
args.len() == 4,
"Supply FIXTURE MODEL_DIR WORKER OUTPUT_JSON; held-out execution is unsupported"
args.len() == 4 || (args.len() == 6 && args[4] == "--held-out-policy"),
"Supply FIXTURE MODEL_DIR WORKER OUTPUT_JSON [--held-out-policy FROZEN_POLICY]"
);
ensure!(
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
@@ -100,12 +109,27 @@ fn main() -> Result<()> {
let models = PathBuf::from(&args[1]).canonicalize()?;
let worker = PathBuf::from(&args[2]).canonicalize()?;
let output = PathBuf::from(&args[3]);
let policy = if args.len() == 6 {
let path = PathBuf::from(&args[5]);
ensure!(
digest(&path)? == "df0a887b97dd7fda6aeff1b6c966fe7558521f40dbe1c911e91cfd312dca7461",
"Policy differs from frozen pre-evaluation hypothesis"
);
Some(serde_json::from_slice::<Value>(&fs::read(path)?)?)
} else {
None
};
let split = if policy.is_some() {
"held_out"
} else {
"calibration"
};
fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&output)
.context("Cannot reserve output; existing evidence is never overwritten")?;
let mut report = json!({"complete":false,"phase":"verifying","split":"calibration","held_out_evaluated":false,"cpu_threads":1,"rows":[]});
let mut report = json!({"complete":false,"phase":"verifying","split":split,"held_out_evaluated":policy.is_some(),"policy":policy,"cpu_threads":1,"rows":[]});
checkpoint(&output, &report)?;
let fixture_path = fixture_root.join("fixture.json");
let fixture: Value = serde_json::from_slice(&fs::read(&fixture_path)?)?;
@@ -119,7 +143,11 @@ fn main() -> Result<()> {
lock["sha256"]["fixture.json"] == digest(&fixture_path)?,
"Frozen fixture changed"
);
let queries = calibration_queries(&fixture)?;
let queries = if policy.is_some() {
split_queries(&fixture, split)?
} else {
calibration_queries(&fixture)?
};
let manifest_path =
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json");
let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
@@ -145,7 +173,7 @@ fn main() -> Result<()> {
let temporary = tempfile::tempdir()?;
let corpus = temporary.path().join("corpus");
fs::create_dir(&corpus)?;
// Index both splits as the fixed distractor library, but execute calibration queries only.
// Index both splits as the fixed distractor library, execute only the selected split.
for asset in fixture["assets"].as_array().context("Missing assets")? {
let source = fixture_root
.join(asset["path"].as_str().context("Missing asset path")?)
@@ -189,18 +217,19 @@ fn main() -> Result<()> {
let query_startup_ms = start.elapsed().as_secs_f64() * 1000.0;
report["worker_info"] = json!(query_session.worker_info());
report["query_startup_ms"] = json!(query_startup_ms);
report["phase"] = json!("calibration_queries");
report["phase"] = json!(format!("{split}_queries"));
checkpoint(&output, &report)?;
for query in queries {
ensure!(
query["split"] == "calibration",
"Held-out execution forbidden"
);
ensure!(query["split"] == split, "Query outside selected split");
let text = query["query"].as_str().context("Missing query text")?;
let code_intent = query["id"]
.as_str()
.context("Missing query ID")?
.starts_with("pattern-");
.starts_with(if split == "held_out" {
"version-"
} else {
"pattern-"
});
for code_only in [false, true]
.into_iter()
.filter(|code_only| !*code_only || code_intent)
@@ -215,7 +244,15 @@ fn main() -> Result<()> {
let semantic_hits = query_session.search_semantic(text, 80, &options)?;
let semantic_ms = start.elapsed().as_secs_f64() * 1000.0;
let hybrid_hits = query_session.search_measured(text, 20, &options, true)?;
let scores = json!({"lexical":score(&lexical_hits.results,query,&fixture)?,"semantic":score(&semantic_hits,query,&fixture)?,"hybrid":score(&hybrid_hits.results,query,&fixture)?});
let retained: Vec<_> = semantic_hits
.iter()
.filter(|hit| {
policy
.as_ref()
.is_none_or(|p| hit.score >= p["semantic_cosine_minimum"].as_f64().unwrap())
})
.collect();
let scores = json!({"lexical":score(&lexical_hits.results,query,&fixture)?,"semantic":score(&semantic_hits,query,&fixture)?,"hybrid":score(&hybrid_hits.results,query,&fixture)?,"hypothesis_semantic":score(retained,query,&fixture)?});
report["rows"].as_array_mut().context("Missing output rows")?.push(json!({"query":query,"scope":if code_only {"code"} else {"all_files"},"lexical":lexical_hits,"semantic_raw":semantic_hits,"semantic_ms":semantic_ms,"hybrid":hybrid_hits,"scores":scores,"order_note":"lexical then semantic then hybrid; hybrid reuses the preceding query-vector cache. Raw semantic scores are cosine, hybrid scores are RRF, not confidence."}));
checkpoint(&output, &report)?;
}
@@ -262,6 +299,24 @@ mod tests {
assert!(calibration_queries(&json!({"queries":[{"split":"held_out"}]})).is_err());
}
#[test]
fn held_out_selection_is_disjoint_and_has_six_code_requests() {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../../fixtures/relevance-v2/fixture.json");
let fixture: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap();
let selected = split_queries(&fixture, "held_out").unwrap();
assert_eq!(selected.len(), 24);
assert!(selected.iter().all(|q| q["split"] == "held_out"));
assert_eq!(
selected
.iter()
.filter(|q| q["id"].as_str().unwrap().starts_with("version-"))
.count(),
6
);
assert!(split_queries(&fixture, "unknown").is_err());
}
#[test]
fn frozen_calibration_has_six_code_requests_and_disjoint_query_ids() {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))