searchi/crates/searchi-core/tests/retrieval_lanes.rs

use anyhow::Result;
use searchi_core::{SearchIndex, SearchOptions};
use std::fs;

#[test]
fn filters_do_not_hide_late_matches_in_any_lane() -> Result<()> {
    let root = tempfile::tempdir()?;
    let rejected = root.path().join("early");
    let wanted = root.path().join("late");
    fs::create_dir(&rejected)?;
    fs::create_dir(&wanted)?;
    let db = root.path().join("index.db");
    let mut index = SearchIndex::open(&db)?;
    for number in 0..2000 {
        fs::write(
            rejected.join(format!("shared_{number:04}.txt")),
            "shared retrieval term",
        )?;
    }
    index.index_folder(&rejected)?;
    fs::write(
        wanted.join("shared_last.md"),
        "shared retrieval term unique-evidence",
    )?;
    index.index_folder(&wanted)?;
    let options = SearchOptions {
        folder: Some(wanted.clone()),
        extension: Some("md".into()),
        ..Default::default()
    };
    assert_eq!(
        index.search_with_options("shared", 1, &options)?[0].filename,
        "shared_last.md"
    );
    assert_eq!(
        index.search_with_options("retrieval", 1, &options)?[0].filename,
        "shared_last.md"
    );
    assert_eq!(
        index.search_with_options("ha", 1, &options)?[0].filename,
        "shared_last.md"
    );
    let result = index.search_with_options("unique-evidence", 1, &options)?;
    assert!(result[0].source.is_some());
    assert!(result[0].snippet.contains("unique-evidence"));
    Ok(())
}

#[test]
fn exact_priority_literal_identifiers_and_index_lifecycle() -> Result<()> {
    let root = tempfile::tempdir()?;
    fs::create_dir(root.path().join("contracts"))?;
    fs::write(
        root.path().join("name_[%].txt"),
        "identifier ABC-123 and term",
    )?;
    fs::write(root.path().join("name_[%].txt.backup"), "term")?;
    fs::write(
        root.path().join("contracts").join("other.md"),
        "name identifier ABC-123",
    )?;
    fs::write(root.path().join("annual_report.md"), "unrelated text")?;
    fs::write(root.path().join("abc_missing_xyz.txt"), "unrelated text")?;
    let mut index = SearchIndex::open(":memory:")?;
    index.index_folder(root.path())?;
    let result = index.search("NAME_[%].TXT", 10)?;
    assert_eq!(result[0].match_kind, "exact_filename");
    assert_eq!(result[1].match_kind, "filename");
    assert_eq!(index.search("contracts", 1)?[0].match_kind, "path");
    assert_eq!(index.search("ABC-123", 10)?.len(), 2);
    assert_eq!(
        index.search("annual report", 1)?[0].filename,
        "annual_report.md"
    );
    assert!(index.search("abc_xyz", 10)?.is_empty());
    let first = index
        .search("name", 10)?
        .into_iter()
        .map(|row| row.path)
        .collect::<Vec<_>>();
    let second = index
        .search("name", 10)?
        .into_iter()
        .map(|row| row.path)
        .collect::<Vec<_>>();
    assert_eq!(first, second);
    fs::rename(
        root.path().join("name_[%].txt"),
        root.path().join("replacement.txt"),
    )?;
    fs::remove_file(root.path().join("name_[%].txt.backup"))?;
    index.index_folder(root.path())?;
    assert!(index.search("name_[%].txt", 10)?.is_empty());
    assert_eq!(index.search("placement", 1)?[0].filename, "replacement.txt");
    Ok(())
}

#[test]
fn v4_migration_backfills_names_and_preserves_passage_ids() -> Result<()> {
    let root = tempfile::tempdir()?;
    let db = root.path().join("old.db");
    let connection = rusqlite::Connection::open(&db)?;
    connection.execute_batch("CREATE TABLE files(id INTEGER PRIMARY KEY,path TEXT NOT NULL UNIQUE,
        root TEXT NOT NULL,filename TEXT NOT NULL,content TEXT NOT NULL,size INTEGER NOT NULL,mtime INTEGER NOT NULL,
        extension TEXT NOT NULL,extraction_error TEXT);
        CREATE VIRTUAL TABLE search USING fts5(filename,path,content,content='files',content_rowid='id',tokenize='unicode61');
        CREATE VIRTUAL TABLE passages USING fts5(text,file_path UNINDEXED,kind UNINDEXED,number UNINDEXED);
        INSERT INTO files VALUES(41,'/fixture/report.pdf','/fixture','report.pdf','preserved evidence',18,123,'pdf',NULL);
        INSERT INTO search(search) VALUES('rebuild');
        INSERT INTO passages(rowid,text,file_path,kind,number) VALUES(88,'preserved evidence','/fixture/report.pdf','page',7);
        PRAGMA user_version=4;")?;
    drop(connection);
    let index = SearchIndex::open(&db)?;
    assert_eq!(index.search("port.p", 1)?[0].filename, "report.pdf");
    assert_eq!(
        index.search("evidence", 1)?[0]
            .source
            .as_ref()
            .unwrap()
            .number,
        7
    );
    assert_eq!(index.passage_batch(0, 1)?[0].id, 88);
    Ok(())
}

#[test]
fn background_statistics_preserve_results_across_reopen_and_changes() -> Result<()> {
    let root = tempfile::tempdir()?;
    let db = root.path().join("index.db");
    fs::write(root.path().join("invoice_1042.txt"), "billing evidence")?;
    let mut index = SearchIndex::open(&db)?;
    index.index_folder(root.path())?;
    assert!(index.prewarm()? > 0);
    drop(index);
    let mut index = SearchIndex::open(&db)?;
    assert_eq!(index.search("1042", 1)?[0].filename, "invoice_1042.txt");
    fs::rename(
        root.path().join("invoice_1042.txt"),
        root.path().join("new_pattern.txt"),
    )?;
    index.index_folder(root.path())?;
    // Stale statistics affect only candidate cost, never substring correctness.
    assert_eq!(index.search("pattern", 1)?[0].filename, "new_pattern.txt");
    assert!(index.search("1042", 1)?.is_empty());
    index.prewarm()?;
    assert_eq!(index.search("pattern", 1)?[0].filename, "new_pattern.txt");
    Ok(())
}

#[test]
fn positional_candidates_preserve_repeated_identifiers_unicode_and_path_matches() -> Result<()> {
    let root = tempfile::tempdir()?;
    let nested = root.path().join("folder_0000001_rare");
    fs::create_dir(&nested)?;
    for (name, text) in [
        ("ζαζα_0000001_rare.txt", "first source"),
        ("ζαζα_0000101_rare.txt", "second source"),
        ("backup_ζαζα_0000001_rare.txt", "backup source"),
    ] {
        fs::write(root.path().join(name), text)?;
    }
    fs::write(nested.join("other.md"), "path source")?;
    let mut index = SearchIndex::open(":memory:")?;
    index.index_folder(root.path())?;
    index.prewarm()?;
    let matches = index.search("0000001_rare", 20)?;
    assert_eq!(matches.len(), 3);
    assert_eq!(matches[0].match_kind, "filename");
    assert_eq!(matches[1].match_kind, "filename");
    assert_eq!(matches[2].match_kind, "path");
    assert_eq!(
        index.search("ζαζα_0000101_rare", 1)?[0].filename,
        "ζαζα_0000101_rare.txt"
    );
    let filtered = SearchOptions {
        folder: Some(nested),
        extension: Some("md".into()),
        ..Default::default()
    };
    assert_eq!(
        index.search_with_options("0000001_rare", 1, &filtered)?[0].filename,
        "other.md"
    );
    assert!(index.search("0000010_rare", 20)?.is_empty());
    Ok(())
}

#[test]
fn candidate_first_content_matches_complete_filtered_ranking_and_sources() -> Result<()> {
    let root = tempfile::tempdir()?;
    fs::write(root.path().join("needle.txt"), "needle once")?;
    for number in 0..50 {
        let text = format!(
            "{} background evidence {}",
            "needle ".repeat(number % 7 + 1),
            "filler ".repeat(number % 5)
        );
        fs::write(root.path().join(format!("notes_{number:03}.md")), text)?;
    }
    let mut index = SearchIndex::open(":memory:")?;
    index.index_folder(root.path())?;
    index.prewarm()?;
    let fast = index.search("needle", 20)?;
    // This nonrestrictive filter selects the normal complete filtered lane.
    let filtered = index.search_with_options(
        "needle",
        20,
        &SearchOptions {
            min_size: Some(0),
            ..Default::default()
        },
    )?;
    assert_eq!(fast.len(), 20);
    assert_eq!(filtered.len(), 20);
    for (fast, filtered) in fast.iter().zip(&filtered) {
        assert_eq!(fast.path, filtered.path);
        assert_eq!(fast.match_kind, filtered.match_kind);
        assert!((fast.score - filtered.score).abs() < 1e-12);
        assert_eq!(fast.snippet, filtered.snippet);
        assert_eq!(
            fast.source.as_ref().unwrap().number,
            filtered.source.as_ref().unwrap().number
        );
    }
    assert_eq!(fast[0].match_kind, "filename");
    Ok(())
}