perf: index filename candidates and avoid full-corpus retrieval scans

crates/searchi-core/tests/retrieval_lanes.rs

@@ -0,0 +1,125 @@
+use anyhow::Result;
+use searchi_core::{SearchIndex, SearchOptions};
+use std::fs;
+
+#[test]
+fn filters_do_not_hide_late_matches_in_any_lane() -> Result<()> {
+    let root = tempfile::tempdir()?;
+    let rejected = root.path().join("early");
+    let wanted = root.path().join("late");
+    fs::create_dir(&rejected)?;
+    fs::create_dir(&wanted)?;
+    let db = root.path().join("index.db");
+    let mut index = SearchIndex::open(&db)?;
+    for number in 0..2000 {
+        fs::write(
+            rejected.join(format!("shared_{number:04}.txt")),
+            "shared retrieval term",
+        )?;
+    }
+    index.index_folder(&rejected)?;
+    fs::write(
+        wanted.join("shared_last.md"),
+        "shared retrieval term unique-evidence",
+    )?;
+    index.index_folder(&wanted)?;
+    let options = SearchOptions {
+        folder: Some(wanted.clone()),
+        extension: Some("md".into()),
+        ..Default::default()
+    };
+    assert_eq!(
+        index.search_with_options("shared", 1, &options)?[0].filename,
+        "shared_last.md"
+    );
+    assert_eq!(
+        index.search_with_options("retrieval", 1, &options)?[0].filename,
+        "shared_last.md"
+    );
+    assert_eq!(
+        index.search_with_options("ha", 1, &options)?[0].filename,
+        "shared_last.md"
+    );
+    let result = index.search_with_options("unique-evidence", 1, &options)?;
+    assert!(result[0].source.is_some());
+    assert!(result[0].snippet.contains("unique-evidence"));
+    Ok(())
+}
+
+#[test]
+fn exact_priority_literal_identifiers_and_index_lifecycle() -> Result<()> {
+    let root = tempfile::tempdir()?;
+    fs::create_dir(root.path().join("contracts"))?;
+    fs::write(
+        root.path().join("name_[%].txt"),
+        "identifier ABC-123 and term",
+    )?;
+    fs::write(root.path().join("name_[%].txt.backup"), "term")?;
+    fs::write(
+        root.path().join("contracts").join("other.md"),
+        "name identifier ABC-123",
+    )?;
+    fs::write(root.path().join("annual_report.md"), "unrelated text")?;
+    fs::write(root.path().join("abc_missing_xyz.txt"), "unrelated text")?;
+    let mut index = SearchIndex::open(":memory:")?;
+    index.index_folder(root.path())?;
+    let result = index.search("NAME_[%].TXT", 10)?;
+    assert_eq!(result[0].match_kind, "exact_filename");
+    assert_eq!(result[1].match_kind, "filename");
+    assert_eq!(index.search("contracts", 1)?[0].match_kind, "path");
+    assert_eq!(index.search("ABC-123", 10)?.len(), 2);
+    assert_eq!(
+        index.search("annual report", 1)?[0].filename,
+        "annual_report.md"
+    );
+    assert!(index.search("abc_xyz", 10)?.is_empty());
+    let first = index
+        .search("name", 10)?
+        .into_iter()
+        .map(|row| row.path)
+        .collect::<Vec<_>>();
+    let second = index
+        .search("name", 10)?
+        .into_iter()
+        .map(|row| row.path)
+        .collect::<Vec<_>>();
+    assert_eq!(first, second);
+    fs::rename(
+        root.path().join("name_[%].txt"),
+        root.path().join("replacement.txt"),
+    )?;
+    fs::remove_file(root.path().join("name_[%].txt.backup"))?;
+    index.index_folder(root.path())?;
+    assert!(index.search("name_[%].txt", 10)?.is_empty());
+    assert_eq!(index.search("placement", 1)?[0].filename, "replacement.txt");
+    Ok(())
+}
+
+#[test]
+fn v4_migration_backfills_names_and_preserves_passage_ids() -> Result<()> {
+    let root = tempfile::tempdir()?;
+    let db = root.path().join("old.db");
+    let connection = rusqlite::Connection::open(&db)?;
+    connection.execute_batch("CREATE TABLE files(id INTEGER PRIMARY KEY,path TEXT NOT NULL UNIQUE,
+        root TEXT NOT NULL,filename TEXT NOT NULL,content TEXT NOT NULL,size INTEGER NOT NULL,mtime INTEGER NOT NULL,
+        extension TEXT NOT NULL,extraction_error TEXT);
+        CREATE VIRTUAL TABLE search USING fts5(filename,path,content,content='files',content_rowid='id',tokenize='unicode61');
+        CREATE VIRTUAL TABLE passages USING fts5(text,file_path UNINDEXED,kind UNINDEXED,number UNINDEXED);
+        INSERT INTO files VALUES(41,'/fixture/report.pdf','/fixture','report.pdf','preserved evidence',18,123,'pdf',NULL);
+        INSERT INTO search(search) VALUES('rebuild');
+        INSERT INTO passages(rowid,text,file_path,kind,number) VALUES(88,'preserved evidence','/fixture/report.pdf','page',7);
+        PRAGMA user_version=4;")?;
+    drop(connection);
+    let index = SearchIndex::open(&db)?;
+    assert_eq!(index.search("port.p", 1)?[0].filename, "report.pdf");
+    assert_eq!(
+        index.search("evidence", 1)?[0]
+            .source
+            .as_ref()
+            .unwrap()
+            .number,
+        7
+    );
+    assert_eq!(index.passage_batch(0, 1)?[0].id, 88);
+    Ok(())
+}