karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Distinguish main matches from related suggestions and ship 0.1.9

Browse Source
This commit is contained in:
karl committed 2026-10-14 12:45:32 +02:00
1 parent a8ba594d0e
commit b0bdfc7816
40 files changed
+2463 -185
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/relevance-eval/Cargo.lock
Generated
+16
View File
@@ -869,13 +869,16 @@ name = "searchi-core"
version = "0.1.0"
dependencies = [
"anyhow",
"libc",
"lopdf",
"quick-xml",
"rusqlite",
"searchi-media",
"serde",
"serde_json",
"sha2 0.10.9",
"walkdir",
"windows-sys",
"zip",
]
@@ -896,8 +899,10 @@ name = "searchi-relevance-eval"
version = "0.1.0"
dependencies = [
"anyhow",
"rusqlite",
"searchi-core",
"searchi-service",
"searchi-speech",
"serde_json",
"sha2 0.10.9",
"tempfile",
@@ -926,11 +931,22 @@ dependencies = [
"searchi-core",
"searchi-media",
"searchi-semantic",
"searchi-speech",
"serde",
"sha2 0.10.9",
"usearch",
]
[[package]]
name = "searchi-speech"
version = "0.1.0"
dependencies = [
"anyhow",
"serde",
"serde_json",
"sha2 0.10.9",
]
[[package]]
name = "serde"
version = "1.0.229"
tools/relevance-eval/Cargo.toml
+2
View File
@@ -12,4 +12,6 @@ serde_json = "1"
sha2 = "0.10"
searchi-core = { path = "../../crates/searchi-core" }
searchi-service = { path = "../../crates/searchi-service" }
searchi-speech = { path = "../../crates/searchi-speech" }
rusqlite = { version = "0.32", features = ["bundled"] }
tempfile = "3"
tools/relevance-eval/README.md
+40
View File
@@ -1,5 +1,45 @@
# Frozen relevance diagnostic
## New absence holdout
`--bin absence` is a separate runner for `fixtures/absence-v3`, with exact
fixture and evidence-policy hashes enforced. It does not change the historical
v2 runner or its held-out gate. It materializes the 36 frozen source files into
an owned persistent profile, hashes existing copies before reuse, runs actual
Google media/text indexing with CPU speech recognition and reports per-asset
extraction errors and active semantic channels. A failed extraction or absent
required channel must not be described as successful abstention.
Set `SEARCHI_FORCE_CPU=1`, `SEARCHI_EXTRACT_WORKER`, `SEARCHI_GEMMA_MODEL_DIR`,
`SEARCHI_GEMMA_WORKER`, `SEARCHI_TEST_FFMPEG`, `SEARCHI_TEST_FFPROBE`,
`SEARCHI_SPEECH_WORKER`, `SEARCHI_SPEECH_BASE` and `SEARCHI_SPEECH_VAD` to verified
local assets. Reserve the shared CPU slot before running:
```powershell
cargo test --manifest-path tools/relevance-eval/Cargo.toml --bin absence --locked
cargo build --release --manifest-path tools/relevance-eval/Cargo.toml --bin absence --locked
tools/relevance-eval/target/release/absence.exe C:/path/to/searchi C:/path/to/searchi/.searchi/qa/absence-v3-runtime C:/path/to/searchi/.searchi/qa/absence-v3-results.json
```
Existing evidence outputs are never overwritten. Raw routed cosine results,
current primary and related results, and a replay of the previously shipped
leading-cluster hybrid rule are retained. The replay uses the actual vectors
and keyword candidates from this run; it is not execution or timing of an old
binary. Its fixture queries contain no identifier-specific search; it does not
replay the service's identifier promotion. Current search runs through the
shared service, including strict code scope. Results are capped at 20, with an
80-candidate budget. Primary absence counts and primary/accessibly retained
positive recall are separate; related suggestions are not claimed to be exact
answers. English/German and category metrics are reported separately. Four
partial requests are excluded from strict positive/absence counts.
Calls run lexical, then raw semantic, then current hybrid. Hybrid reuses the
query-vector cache, so these timings do not measure cold first user input. Do
not tune the frozen policy on this holdout. See the fixture README for label
and corpus limitations.
## Historical v2 diagnostic
This internal runner uses searchi's shared Rust engine and genuine Google
EmbeddingGemma 2. By default it indexes all 14 frozen source files as the fixed
distractor library and evaluates only the 24 calibration queries. Held-out
tools/relevance-eval/policies/evidence-v1.json
+20
View File
@@ -0,0 +1,20 @@
{
"id": "semantic-evidence-presentation-v1",
"frozen_before_new_query_holdout": true,
"holdout_fixture_sha256": "12354a55c1e5f1efe3437e653c074d247374852a420329db8d500771c4a24f53",
"main_semantic_cosine_minimum": 0.70,
"keep_existing_leading_cluster_rule": true,
"weak_and_tail_candidates": "related suggestions, retained and explicitly distinguishable",
"lexical_candidates": "retain existing filename/keyword priority and match kind",
"raw_semantic_diagnostic_api": "unchanged",
"decision_is": "presentation evidence, not probability or proof of artifact presence/absence",
"development_tradeoff": {
"text_all_positive_tops_main": "31/32",
"text_all_absent_tops_main": "0/16",
"code_positive_tops_main": "8/8",
"code_absent_tops_main": "0/4",
"media_positive_tops_main": "49/55, top-file correctness separately required",
"media_absent_tops_main": "1/6"
},
"rejected": ["hard deletion below a universal cutoff", "confidence percentages", "query/file-specific exceptions", "definite no-content claim from similarity alone"]
}
tools/relevance-eval/src/bin/absence.rs
+406
View File
@@ -0,0 +1,406 @@
//! New frozen holdout: actual CPU retrieval, distinct primary/related evidence.
use anyhow::{ensure, Context, Result};
use searchi_core::{SearchIndex, SearchOptions, SearchResult};
use searchi_service::{is_related_result, FfmpegTools, HybridSearch, MediaLimits};
use searchi_speech::{Device, SpeechWorker};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use std::{
collections::{BTreeMap, HashMap, HashSet},
fs,
io::{Read, Write},
path::{Path, PathBuf},
time::Instant,
};
const FIXTURE: &str = "12354a55c1e5f1efe3437e653c074d247374852a420329db8d500771c4a24f53";
const POLICY: &str = "d3d4e437a54d1d62a70bb09c2882db6362a2566525a54f92d5fdc7fdfb91e26c";
fn digest(path: &Path) -> Result<String> {
let mut file = fs::File::open(path)?;
let mut hash = Sha256::new();
let mut bytes = [0; 65536];
loop {
let count = file.read(&mut bytes)?;
if count == 0 {
break;
}
hash.update(&bytes[..count]);
}
Ok(format!("{:x}", hash.finalize()))
}
fn checkpoint(path: &Path, report: &Value) -> Result<()> {
let mut staged = tempfile::NamedTempFile::new_in(path.parent().context("Output parent")?)?;
staged.write_all(&serde_json::to_vec_pretty(report)?)?;
staged.as_file().sync_all()?;
staged.persist(path)?;
Ok(())
}
fn env_path(key: &str) -> Result<PathBuf> {
PathBuf::from(std::env::var_os(key).with_context(|| format!("Set {key}"))?)
.canonicalize()
.map_err(Into::into)
}
fn copy_result(hit: &SearchResult) -> SearchResult {
SearchResult {
path: hit.path.clone(),
filename: hit.filename.clone(),
score: hit.score,
snippet: hit.snippet.clone(),
match_kind: hit.match_kind.clone(),
source: hit.source.clone(),
extraction_error: hit.extraction_error.clone(),
}
}
fn score(hits: &[SearchResult], query: &Value, assets: &[Value]) -> Result<Value> {
let expected = query["expected_ids"].as_array().context("Expected ids")?;
let mut ids = Vec::new();
for hit in hits {
let asset = assets
.iter()
.find(|a| {
Path::new(a["path"].as_str().unwrap_or(""))
.file_name()
.is_some_and(|n| n == hit.filename.as_str())
})
.context("Result outside fixture")?;
if !ids.contains(&asset["id"]) {
ids.push(asset["id"].clone());
}
}
Ok(
json!({"asset_ranking":ids,"top1_relevant":ids.first().is_some_and(|id|expected.contains(id)),"recall_at_5":if expected.is_empty(){None}else{Some(ids.iter().take(5).filter(|id|expected.contains(id)).count() as f64 / expected.len() as f64)},"any_result":!ids.is_empty()}),
)
}
// Replay the previously shipped 0.07-gap policy over actual retrieved vectors.
// This is labelled a replay, not a separately measured old binary.
fn baseline(mut semantic: Vec<SearchResult>, lexical: Vec<SearchResult>) -> Vec<SearchResult> {
let mut gaps: Vec<_> = semantic
.windows(2)
.enumerate()
.map(|(i, p)| (i + 1, p[0].score - p[1].score))
.collect();
gaps.sort_by(|a, b| b.1.total_cmp(&a.1));
if let Some(&(boundary, gap)) = gaps.first() {
if gap > 0.07 && gap - gaps.get(1).map_or(0.0, |p| p.1) > 2.0 * f32::EPSILON as f64 {
semantic.truncate(boundary);
}
}
let mut results = HashMap::new();
for (rank, mut hit) in lexical.into_iter().enumerate() {
hit.score = if hit.match_kind == "filename" {
1.15
} else {
1.0
} / (61.0 + rank as f64);
results.insert(hit.path.clone(), hit);
}
for (rank, mut hit) in semantic.into_iter().enumerate() {
let contribution = 1.0 / (61.0 + rank as f64);
if let Some(existing) = results.get_mut(&hit.path) {
existing.score += contribution;
if existing.match_kind == "content" {
existing.match_kind = "hybrid".into();
}
} else {
hit.score = contribution;
results.insert(hit.path.clone(), hit);
}
}
let tier = |hit: &SearchResult| match hit.match_kind.as_str() {
"exact_filename" => 0,
"exact_identifier" => 1,
_ => 2,
};
let mut results: Vec<_> = results.into_values().collect();
results.sort_by(|a, b| {
tier(a)
.cmp(&tier(b))
.then_with(|| b.score.total_cmp(&a.score))
.then_with(|| a.path.cmp(&b.path))
});
results.truncate(20);
results
}
fn coverage(db: &Path, assets: &[Value], corpus: &Path) -> Result<Value> {
let connection =
rusqlite::Connection::open_with_flags(db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let mut rows = Vec::new();
for asset in assets {
let path = corpus.join(
Path::new(asset["path"].as_str().context("Path")?)
.file_name()
.context("Filename")?,
);
let path = path.to_string_lossy().to_string();
let extraction = connection.query_row(
"SELECT extraction_error,length(content),extracted_sha256 FROM files WHERE path=?1",
[&path],
|r| {
Ok((
r.get::<_, Option<String>>(0)?,
r.get::<_, usize>(1)?,
r.get::<_, Option<String>>(2)?,
))
},
)?;
let mut channels = BTreeMap::<String, u64>::new();
let mut statement=connection.prepare("SELECT source_type,count(*) FROM semantic_chunks WHERE generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1) AND path=?1 GROUP BY source_type")?;
for row in statement.query_map([&path], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, u64>(1)?))
})? {
let (k, v) = row?;
channels.insert(k, v);
}
rows.push(json!({"asset_id":asset["id"],"kind":asset["kind"],"extraction_error":extraction.0,"text_characters":extraction.1,"extracted_sha256":extraction.2,"semantic_channels":channels,"has_semantic_chunks":!channels.is_empty()}));
}
Ok(json!(rows))
}
fn summaries(rows: &[Value]) -> Value {
let mut groups = BTreeMap::<String, Vec<&Value>>::new();
for row in rows {
for key in [
"all".to_owned(),
format!("language:{}", row["query"]["language"].as_str().unwrap()),
format!("category:{}", row["query"]["category"].as_str().unwrap()),
] {
groups.entry(key).or_default().push(row);
}
}
let mut output = BTreeMap::new();
for (key, group) in groups {
let positives: Vec<_> = group
.iter()
.filter(|r| r["query"]["label"] == "answerable")
.collect();
let absent: Vec<_> = group
.iter()
.filter(|r| r["query"]["label"] == "absent")
.collect();
let mut metrics = json!({"positive_count":positives.len(),"absent_count":absent.len(),"partial_count":group.iter().filter(|r|r["query"]["label"]=="partial").count()});
for lane in ["raw", "baseline_replay", "primary", "accessible"] {
metrics[format!("{lane}_positive_recall5")] = json!(if positives.is_empty() {
None
} else {
Some(
positives
.iter()
.map(|r| r["scores"][lane]["recall_at_5"].as_f64().unwrap_or(0.0))
.sum::<f64>()
/ positives.len() as f64,
)
});
metrics[format!("{lane}_absent_nonempty")] = json!(absent
.iter()
.filter(|r| r["scores"][lane]["any_result"] == true)
.count());
}
output.insert(key, metrics);
}
json!(output)
}
fn main() -> Result<()> {
ensure!(
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
"CPU only"
);
let args: Vec<_> = std::env::args_os().skip(1).collect();
ensure!(
args.len() == 3,
"Supply REPO_ROOT PROFILE OUTPUT_JSON; runtime paths via SEARCHI_* environment"
);
let root = PathBuf::from(&args[0]).canonicalize()?;
let profile = PathBuf::from(&args[1]);
fs::create_dir_all(&profile)?;
let profile = profile.canonicalize()?;
ensure!(
profile.starts_with(root.join(".searchi/qa")),
"Profile must be owned QA directory"
);
let output = PathBuf::from(&args[2]);
fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&output)?;
let fixture_path = root.join("fixtures/absence-v3/fixture.json");
let policy_path = root.join("tools/relevance-eval/policies/evidence-v1.json");
ensure!(
digest(&fixture_path)? == FIXTURE && digest(&policy_path)? == POLICY,
"Frozen input changed"
);
let fixture: Value = serde_json::from_slice(&fs::read(&fixture_path)?)?;
let assets = fixture["assets"].as_array().context("Assets")?;
let mut report = json!({"complete":false,"fixture_sha256":FIXTURE,"policy_sha256":POLICY,"cpu_threads":1,"rows":[],"phase":"verification"});
checkpoint(&output, &report)?;
let corpus = profile.join("corpus");
fs::create_dir_all(&corpus)?;
let mut names = HashSet::new();
for asset in assets {
let source = root
.join(asset["path"].as_str().context("Asset path")?)
.canonicalize()?;
ensure!(source.starts_with(&root), "Escaped source");
ensure!(
digest(&source)? == asset["sha256"]
&& fs::metadata(&source)?.len() == asset["bytes"].as_u64().context("Bytes")?,
"Source changed"
);
let name = source.file_name().context("Basename")?;
ensure!(names.insert(name.to_owned()), "Duplicate basename");
let destination = corpus.join(name);
if destination.exists() {
ensure!(
digest(&destination)? == asset["sha256"],
"Existing corpus changed"
);
} else {
let mut dest = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&destination)?;
let mut input = fs::File::open(&source)?;
std::io::copy(&mut input, &mut dest)?;
dest.sync_all()?;
}
}
ensure!(
fs::read_dir(&corpus)?.count() == assets.len(),
"Unexpected corpus file"
);
let model = env_path("SEARCHI_GEMMA_MODEL_DIR")?;
let worker = env_path("SEARCHI_GEMMA_WORKER")?;
let ffmpeg = env_path("SEARCHI_TEST_FFMPEG")?;
let ffprobe = env_path("SEARCHI_TEST_FFPROBE")?;
let speech = env_path("SEARCHI_SPEECH_WORKER")?;
let base = env_path("SEARCHI_SPEECH_BASE")?;
let vad = env_path("SEARCHI_SPEECH_VAD")?;
let manifest_path = root.join("tools/embeddinggemma2/model-manifest.json");
let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
for asset in manifest["files"].as_array().context("Model files")? {
let path = model.join(asset["path"].as_str().context("Model path")?);
ensure!(
digest(&path)? == asset["sha256"]
&& fs::metadata(path)?.len() == asset["bytes"].as_u64().unwrap(),
"Model bytes changed"
);
}
report["runtime_hashes"] = json!({"runner":digest(&std::env::current_exe()?)?,"worker":digest(&worker)?,"speech_worker":digest(&speech)?,"speech_base":digest(&base)?,"speech_vad":digest(&vad)?,"ffmpeg":digest(&ffmpeg)?,"ffprobe":digest(&ffprobe)?,"model_manifest":digest(&manifest_path)?});
report["source_hashes"] = json!({"service_lib":digest(&root.join("crates/searchi-service/src/lib.rs"))?,"relevance":digest(&root.join("crates/searchi-service/src/relevance.rs"))?,"routing":digest(&root.join("crates/searchi-service/src/routing.rs"))?});
report["assets"] = fixture["assets"].clone();
report["phase"] = json!("indexing");
checkpoint(&output, &report)?;
let db = profile.join("index.sqlite");
let mut lexical = SearchIndex::open(&db)?;
let now = Instant::now();
report["lexical_report"] = json!(lexical.index_folder(&corpus)?);
report["lexical_ms"] = json!(now.elapsed().as_secs_f64() * 1000.0);
checkpoint(&output, &report)?;
let now = Instant::now();
let mut indexer = HybridSearch::open(&db, &model, &worker, 1)?.with_speech_provider(
Box::new(SpeechWorker::load(&speech, &base, &vad, Device::Cpu)?),
)?;
let scratch = profile.join("decoded-media");
fs::create_dir_all(&scratch)?;
let limits = MediaLimits {
scratch_directory: Some(scratch),
..Default::default()
};
report["index_report"] =
json!(indexer.index_embeddings_with_media(&FfmpegTools { ffmpeg, ffprobe }, &limits)?);
report["semantic_index_ms"] = json!(now.elapsed().as_secs_f64() * 1000.0);
drop(indexer);
report["coverage"] = coverage(&db, assets, &corpus)?;
checkpoint(&output, &report)?;
let now = Instant::now();
let mut search = HybridSearch::open_query(&db, &model, &worker, 1)?;
search.warmup()?;
report["startup_ms"] = json!(now.elapsed().as_secs_f64() * 1000.0);
report["worker_info"] = json!(search.worker_info());
report["phase"] = json!("queries");
checkpoint(&output, &report)?;
for query in fixture["queries"].as_array().context("Queries")? {
let options = SearchOptions {
code_only: query["scope"] == "code",
..Default::default()
};
let text = query["text"].as_str().context("Text")?;
let lexical_hits = lexical.search_with_options_measured(text, 80, &options, true)?;
let now = Instant::now();
let raw = search.search_semantic(text, 80, &options)?;
let raw_ms = now.elapsed().as_secs_f64() * 1000.0;
let baseline = baseline(
raw.iter().map(copy_result).collect(),
lexical_hits.results.iter().map(copy_result).collect(),
);
let measured = search.search_measured(text, 20, &options, true)?;
let primary: Vec<_> = measured
.results
.iter()
.filter(|r| !is_related_result(r))
.map(copy_result)
.collect();
let related: Vec<_> = measured
.results
.iter()
.filter(|r| is_related_result(r))
.map(copy_result)
.collect();
let scores = json!({"raw":score(&raw,query,assets)?,"baseline_replay":score(&baseline,query,assets)?,"primary":score(&primary,query,assets)?,"related":score(&related,query,assets)?,"accessible":score(&measured.results,query,assets)?});
report["rows"].as_array_mut().unwrap().push(json!({"query":query,"lexical":lexical_hits,"raw":raw,"raw_ms":raw_ms,"baseline_replay":baseline,"current":measured,"scores":scores}));
checkpoint(&output, &report)?;
}
report["summary"] = summaries(report["rows"].as_array().unwrap());
report["complete"] = json!(true);
report["phase"] = json!("complete");
report["order_note"]=json!("lexical then raw then hybrid; hybrid reuses query vector. Replay baseline has no separate runtime timing. Scores not confidence; coverage failures must be separated.");
checkpoint(&output, &report)?;
println!("{}", report["summary"]);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn hit(name: &str, score: f64) -> SearchResult {
SearchResult {
path: name.into(),
filename: name.into(),
snippet: String::new(),
score,
match_kind: "semantic".into(),
source: None,
extraction_error: None,
}
}
#[test]
fn baseline_gap_replay_keeps_best_and_flat_neighbors() {
assert_eq!(
baseline(vec![hit("a", 0.82), hit("b", 0.66), hit("c", 0.63)], vec![]).len(),
1
);
assert_eq!(
baseline(vec![hit("a", 0.60), hit("b", 0.58), hit("c", 0.56)], vec![]).len(),
3
);
assert_eq!(
baseline(vec![hit("a", 0.9), hit("b", 0.8), hit("c", 0.7)], vec![]).len(),
3
);
}
#[test]
fn baseline_preserves_lexical_after_semantic_split() {
let mut lexical = hit("c", 0.0);
lexical.match_kind = "exact_filename".into();
let result = baseline(
vec![hit("a", 0.82), hit("b", 0.66), hit("c", 0.63)],
vec![lexical],
);
assert_eq!(result.len(), 2);
assert_eq!(result[0].filename, "c");
}
#[test]
fn empty_absent_is_not_positive_recall() {
let scores = score(&[], &json!({"expected_ids":[]}), &[]).unwrap();
assert!(scores["recall_at_5"].is_null());
assert_eq!(scores["any_result"], false);
}
}