Distinguish main matches from related suggestions and ship 0.1.9

tools/relevance-eval/src/bin/absence.rs

@@ -0,0 +1,406 @@
+//! New frozen holdout: actual CPU retrieval, distinct primary/related evidence.
+use anyhow::{ensure, Context, Result};
+use searchi_core::{SearchIndex, SearchOptions, SearchResult};
+use searchi_service::{is_related_result, FfmpegTools, HybridSearch, MediaLimits};
+use searchi_speech::{Device, SpeechWorker};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{
+    collections::{BTreeMap, HashMap, HashSet},
+    fs,
+    io::{Read, Write},
+    path::{Path, PathBuf},
+    time::Instant,
+};
+
+const FIXTURE: &str = "12354a55c1e5f1efe3437e653c074d247374852a420329db8d500771c4a24f53";
+const POLICY: &str = "d3d4e437a54d1d62a70bb09c2882db6362a2566525a54f92d5fdc7fdfb91e26c";
+fn digest(path: &Path) -> Result<String> {
+    let mut file = fs::File::open(path)?;
+    let mut hash = Sha256::new();
+    let mut bytes = [0; 65536];
+    loop {
+        let count = file.read(&mut bytes)?;
+        if count == 0 {
+            break;
+        }
+        hash.update(&bytes[..count]);
+    }
+    Ok(format!("{:x}", hash.finalize()))
+}
+fn checkpoint(path: &Path, report: &Value) -> Result<()> {
+    let mut staged = tempfile::NamedTempFile::new_in(path.parent().context("Output parent")?)?;
+    staged.write_all(&serde_json::to_vec_pretty(report)?)?;
+    staged.as_file().sync_all()?;
+    staged.persist(path)?;
+    Ok(())
+}
+fn env_path(key: &str) -> Result<PathBuf> {
+    PathBuf::from(std::env::var_os(key).with_context(|| format!("Set {key}"))?)
+        .canonicalize()
+        .map_err(Into::into)
+}
+fn copy_result(hit: &SearchResult) -> SearchResult {
+    SearchResult {
+        path: hit.path.clone(),
+        filename: hit.filename.clone(),
+        score: hit.score,
+        snippet: hit.snippet.clone(),
+        match_kind: hit.match_kind.clone(),
+        source: hit.source.clone(),
+        extraction_error: hit.extraction_error.clone(),
+    }
+}
+fn score(hits: &[SearchResult], query: &Value, assets: &[Value]) -> Result<Value> {
+    let expected = query["expected_ids"].as_array().context("Expected ids")?;
+    let mut ids = Vec::new();
+    for hit in hits {
+        let asset = assets
+            .iter()
+            .find(|a| {
+                Path::new(a["path"].as_str().unwrap_or(""))
+                    .file_name()
+                    .is_some_and(|n| n == hit.filename.as_str())
+            })
+            .context("Result outside fixture")?;
+        if !ids.contains(&asset["id"]) {
+            ids.push(asset["id"].clone());
+        }
+    }
+    Ok(
+        json!({"asset_ranking":ids,"top1_relevant":ids.first().is_some_and(|id|expected.contains(id)),"recall_at_5":if expected.is_empty(){None}else{Some(ids.iter().take(5).filter(|id|expected.contains(id)).count() as f64 / expected.len() as f64)},"any_result":!ids.is_empty()}),
+    )
+}
+// Replay the previously shipped 0.07-gap policy over actual retrieved vectors.
+// This is labelled a replay, not a separately measured old binary.
+fn baseline(mut semantic: Vec<SearchResult>, lexical: Vec<SearchResult>) -> Vec<SearchResult> {
+    let mut gaps: Vec<_> = semantic
+        .windows(2)
+        .enumerate()
+        .map(|(i, p)| (i + 1, p[0].score - p[1].score))
+        .collect();
+    gaps.sort_by(|a, b| b.1.total_cmp(&a.1));
+    if let Some(&(boundary, gap)) = gaps.first() {
+        if gap > 0.07 && gap - gaps.get(1).map_or(0.0, |p| p.1) > 2.0 * f32::EPSILON as f64 {
+            semantic.truncate(boundary);
+        }
+    }
+    let mut results = HashMap::new();
+    for (rank, mut hit) in lexical.into_iter().enumerate() {
+        hit.score = if hit.match_kind == "filename" {
+            1.15
+        } else {
+            1.0
+        } / (61.0 + rank as f64);
+        results.insert(hit.path.clone(), hit);
+    }
+    for (rank, mut hit) in semantic.into_iter().enumerate() {
+        let contribution = 1.0 / (61.0 + rank as f64);
+        if let Some(existing) = results.get_mut(&hit.path) {
+            existing.score += contribution;
+            if existing.match_kind == "content" {
+                existing.match_kind = "hybrid".into();
+            }
+        } else {
+            hit.score = contribution;
+            results.insert(hit.path.clone(), hit);
+        }
+    }
+    let tier = |hit: &SearchResult| match hit.match_kind.as_str() {
+        "exact_filename" => 0,
+        "exact_identifier" => 1,
+        _ => 2,
+    };
+    let mut results: Vec<_> = results.into_values().collect();
+    results.sort_by(|a, b| {
+        tier(a)
+            .cmp(&tier(b))
+            .then_with(|| b.score.total_cmp(&a.score))
+            .then_with(|| a.path.cmp(&b.path))
+    });
+    results.truncate(20);
+    results
+}
+fn coverage(db: &Path, assets: &[Value], corpus: &Path) -> Result<Value> {
+    let connection =
+        rusqlite::Connection::open_with_flags(db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
+    let mut rows = Vec::new();
+    for asset in assets {
+        let path = corpus.join(
+            Path::new(asset["path"].as_str().context("Path")?)
+                .file_name()
+                .context("Filename")?,
+        );
+        let path = path.to_string_lossy().to_string();
+        let extraction = connection.query_row(
+            "SELECT extraction_error,length(content),extracted_sha256 FROM files WHERE path=?1",
+            [&path],
+            |r| {
+                Ok((
+                    r.get::<_, Option<String>>(0)?,
+                    r.get::<_, usize>(1)?,
+                    r.get::<_, Option<String>>(2)?,
+                ))
+            },
+        )?;
+        let mut channels = BTreeMap::<String, u64>::new();
+        let mut statement=connection.prepare("SELECT source_type,count(*) FROM semantic_chunks WHERE generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1) AND path=?1 GROUP BY source_type")?;
+        for row in statement.query_map([&path], |r| {
+            Ok((r.get::<_, String>(0)?, r.get::<_, u64>(1)?))
+        })? {
+            let (k, v) = row?;
+            channels.insert(k, v);
+        }
+        rows.push(json!({"asset_id":asset["id"],"kind":asset["kind"],"extraction_error":extraction.0,"text_characters":extraction.1,"extracted_sha256":extraction.2,"semantic_channels":channels,"has_semantic_chunks":!channels.is_empty()}));
+    }
+    Ok(json!(rows))
+}
+fn summaries(rows: &[Value]) -> Value {
+    let mut groups = BTreeMap::<String, Vec<&Value>>::new();
+    for row in rows {
+        for key in [
+            "all".to_owned(),
+            format!("language:{}", row["query"]["language"].as_str().unwrap()),
+            format!("category:{}", row["query"]["category"].as_str().unwrap()),
+        ] {
+            groups.entry(key).or_default().push(row);
+        }
+    }
+    let mut output = BTreeMap::new();
+    for (key, group) in groups {
+        let positives: Vec<_> = group
+            .iter()
+            .filter(|r| r["query"]["label"] == "answerable")
+            .collect();
+        let absent: Vec<_> = group
+            .iter()
+            .filter(|r| r["query"]["label"] == "absent")
+            .collect();
+        let mut metrics = json!({"positive_count":positives.len(),"absent_count":absent.len(),"partial_count":group.iter().filter(|r|r["query"]["label"]=="partial").count()});
+        for lane in ["raw", "baseline_replay", "primary", "accessible"] {
+            metrics[format!("{lane}_positive_recall5")] = json!(if positives.is_empty() {
+                None
+            } else {
+                Some(
+                    positives
+                        .iter()
+                        .map(|r| r["scores"][lane]["recall_at_5"].as_f64().unwrap_or(0.0))
+                        .sum::<f64>()
+                        / positives.len() as f64,
+                )
+            });
+            metrics[format!("{lane}_absent_nonempty")] = json!(absent
+                .iter()
+                .filter(|r| r["scores"][lane]["any_result"] == true)
+                .count());
+        }
+        output.insert(key, metrics);
+    }
+    json!(output)
+}
+fn main() -> Result<()> {
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU only"
+    );
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 3,
+        "Supply REPO_ROOT PROFILE OUTPUT_JSON; runtime paths via SEARCHI_* environment"
+    );
+    let root = PathBuf::from(&args[0]).canonicalize()?;
+    let profile = PathBuf::from(&args[1]);
+    fs::create_dir_all(&profile)?;
+    let profile = profile.canonicalize()?;
+    ensure!(
+        profile.starts_with(root.join(".searchi/qa")),
+        "Profile must be owned QA directory"
+    );
+    let output = PathBuf::from(&args[2]);
+    fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(&output)?;
+    let fixture_path = root.join("fixtures/absence-v3/fixture.json");
+    let policy_path = root.join("tools/relevance-eval/policies/evidence-v1.json");
+    ensure!(
+        digest(&fixture_path)? == FIXTURE && digest(&policy_path)? == POLICY,
+        "Frozen input changed"
+    );
+    let fixture: Value = serde_json::from_slice(&fs::read(&fixture_path)?)?;
+    let assets = fixture["assets"].as_array().context("Assets")?;
+    let mut report = json!({"complete":false,"fixture_sha256":FIXTURE,"policy_sha256":POLICY,"cpu_threads":1,"rows":[],"phase":"verification"});
+    checkpoint(&output, &report)?;
+    let corpus = profile.join("corpus");
+    fs::create_dir_all(&corpus)?;
+    let mut names = HashSet::new();
+    for asset in assets {
+        let source = root
+            .join(asset["path"].as_str().context("Asset path")?)
+            .canonicalize()?;
+        ensure!(source.starts_with(&root), "Escaped source");
+        ensure!(
+            digest(&source)? == asset["sha256"]
+                && fs::metadata(&source)?.len() == asset["bytes"].as_u64().context("Bytes")?,
+            "Source changed"
+        );
+        let name = source.file_name().context("Basename")?;
+        ensure!(names.insert(name.to_owned()), "Duplicate basename");
+        let destination = corpus.join(name);
+        if destination.exists() {
+            ensure!(
+                digest(&destination)? == asset["sha256"],
+                "Existing corpus changed"
+            );
+        } else {
+            let mut dest = fs::OpenOptions::new()
+                .write(true)
+                .create_new(true)
+                .open(&destination)?;
+            let mut input = fs::File::open(&source)?;
+            std::io::copy(&mut input, &mut dest)?;
+            dest.sync_all()?;
+        }
+    }
+    ensure!(
+        fs::read_dir(&corpus)?.count() == assets.len(),
+        "Unexpected corpus file"
+    );
+    let model = env_path("SEARCHI_GEMMA_MODEL_DIR")?;
+    let worker = env_path("SEARCHI_GEMMA_WORKER")?;
+    let ffmpeg = env_path("SEARCHI_TEST_FFMPEG")?;
+    let ffprobe = env_path("SEARCHI_TEST_FFPROBE")?;
+    let speech = env_path("SEARCHI_SPEECH_WORKER")?;
+    let base = env_path("SEARCHI_SPEECH_BASE")?;
+    let vad = env_path("SEARCHI_SPEECH_VAD")?;
+    let manifest_path = root.join("tools/embeddinggemma2/model-manifest.json");
+    let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
+    for asset in manifest["files"].as_array().context("Model files")? {
+        let path = model.join(asset["path"].as_str().context("Model path")?);
+        ensure!(
+            digest(&path)? == asset["sha256"]
+                && fs::metadata(path)?.len() == asset["bytes"].as_u64().unwrap(),
+            "Model bytes changed"
+        );
+    }
+    report["runtime_hashes"] = json!({"runner":digest(&std::env::current_exe()?)?,"worker":digest(&worker)?,"speech_worker":digest(&speech)?,"speech_base":digest(&base)?,"speech_vad":digest(&vad)?,"ffmpeg":digest(&ffmpeg)?,"ffprobe":digest(&ffprobe)?,"model_manifest":digest(&manifest_path)?});
+    report["source_hashes"] = json!({"service_lib":digest(&root.join("crates/searchi-service/src/lib.rs"))?,"relevance":digest(&root.join("crates/searchi-service/src/relevance.rs"))?,"routing":digest(&root.join("crates/searchi-service/src/routing.rs"))?});
+    report["assets"] = fixture["assets"].clone();
+    report["phase"] = json!("indexing");
+    checkpoint(&output, &report)?;
+    let db = profile.join("index.sqlite");
+    let mut lexical = SearchIndex::open(&db)?;
+    let now = Instant::now();
+    report["lexical_report"] = json!(lexical.index_folder(&corpus)?);
+    report["lexical_ms"] = json!(now.elapsed().as_secs_f64() * 1000.0);
+    checkpoint(&output, &report)?;
+    let now = Instant::now();
+    let mut indexer = HybridSearch::open(&db, &model, &worker, 1)?.with_speech_provider(
+        Box::new(SpeechWorker::load(&speech, &base, &vad, Device::Cpu)?),
+    )?;
+    let scratch = profile.join("decoded-media");
+    fs::create_dir_all(&scratch)?;
+    let limits = MediaLimits {
+        scratch_directory: Some(scratch),
+        ..Default::default()
+    };
+    report["index_report"] =
+        json!(indexer.index_embeddings_with_media(&FfmpegTools { ffmpeg, ffprobe }, &limits)?);
+    report["semantic_index_ms"] = json!(now.elapsed().as_secs_f64() * 1000.0);
+    drop(indexer);
+    report["coverage"] = coverage(&db, assets, &corpus)?;
+    checkpoint(&output, &report)?;
+    let now = Instant::now();
+    let mut search = HybridSearch::open_query(&db, &model, &worker, 1)?;
+    search.warmup()?;
+    report["startup_ms"] = json!(now.elapsed().as_secs_f64() * 1000.0);
+    report["worker_info"] = json!(search.worker_info());
+    report["phase"] = json!("queries");
+    checkpoint(&output, &report)?;
+    for query in fixture["queries"].as_array().context("Queries")? {
+        let options = SearchOptions {
+            code_only: query["scope"] == "code",
+            ..Default::default()
+        };
+        let text = query["text"].as_str().context("Text")?;
+        let lexical_hits = lexical.search_with_options_measured(text, 80, &options, true)?;
+        let now = Instant::now();
+        let raw = search.search_semantic(text, 80, &options)?;
+        let raw_ms = now.elapsed().as_secs_f64() * 1000.0;
+        let baseline = baseline(
+            raw.iter().map(copy_result).collect(),
+            lexical_hits.results.iter().map(copy_result).collect(),
+        );
+        let measured = search.search_measured(text, 20, &options, true)?;
+        let primary: Vec<_> = measured
+            .results
+            .iter()
+            .filter(|r| !is_related_result(r))
+            .map(copy_result)
+            .collect();
+        let related: Vec<_> = measured
+            .results
+            .iter()
+            .filter(|r| is_related_result(r))
+            .map(copy_result)
+            .collect();
+        let scores = json!({"raw":score(&raw,query,assets)?,"baseline_replay":score(&baseline,query,assets)?,"primary":score(&primary,query,assets)?,"related":score(&related,query,assets)?,"accessible":score(&measured.results,query,assets)?});
+        report["rows"].as_array_mut().unwrap().push(json!({"query":query,"lexical":lexical_hits,"raw":raw,"raw_ms":raw_ms,"baseline_replay":baseline,"current":measured,"scores":scores}));
+        checkpoint(&output, &report)?;
+    }
+    report["summary"] = summaries(report["rows"].as_array().unwrap());
+    report["complete"] = json!(true);
+    report["phase"] = json!("complete");
+    report["order_note"]=json!("lexical then raw then hybrid; hybrid reuses query vector. Replay baseline has no separate runtime timing. Scores not confidence; coverage failures must be separated.");
+    checkpoint(&output, &report)?;
+    println!("{}", report["summary"]);
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    fn hit(name: &str, score: f64) -> SearchResult {
+        SearchResult {
+            path: name.into(),
+            filename: name.into(),
+            snippet: String::new(),
+            score,
+            match_kind: "semantic".into(),
+            source: None,
+            extraction_error: None,
+        }
+    }
+    #[test]
+    fn baseline_gap_replay_keeps_best_and_flat_neighbors() {
+        assert_eq!(
+            baseline(vec![hit("a", 0.82), hit("b", 0.66), hit("c", 0.63)], vec![]).len(),
+            1
+        );
+        assert_eq!(
+            baseline(vec![hit("a", 0.60), hit("b", 0.58), hit("c", 0.56)], vec![]).len(),
+            3
+        );
+        assert_eq!(
+            baseline(vec![hit("a", 0.9), hit("b", 0.8), hit("c", 0.7)], vec![]).len(),
+            3
+        );
+    }
+    #[test]
+    fn baseline_preserves_lexical_after_semantic_split() {
+        let mut lexical = hit("c", 0.0);
+        lexical.match_kind = "exact_filename".into();
+        let result = baseline(
+            vec![hit("a", 0.82), hit("b", 0.66), hit("c", 0.63)],
+            vec![lexical],
+        );
+        assert_eq!(result.len(), 2);
+        assert_eq!(result[0].filename, "c");
+    }
+    #[test]
+    fn empty_absent_is_not_positive_recall() {
+        let scores = score(&[], &json!({"expected_ids":[]}), &[]).unwrap();
+        assert!(scores["recall_at_5"].is_null());
+        assert_eq!(scores["any_result"], false);
+    }
+}