use anyhow::Result; use searchi_core::{SearchIndex, SearchOptions}; use std::fs; #[test] fn filters_do_not_hide_late_matches_in_any_lane() -> Result<()> { let root = tempfile::tempdir()?; let rejected = root.path().join("early"); let wanted = root.path().join("late"); fs::create_dir(&rejected)?; fs::create_dir(&wanted)?; let db = root.path().join("index.db"); let mut index = SearchIndex::open(&db)?; for number in 0..2000 { fs::write( rejected.join(format!("shared_{number:04}.txt")), "shared retrieval term", )?; } index.index_folder(&rejected)?; fs::write( wanted.join("shared_last.md"), "shared retrieval term unique-evidence", )?; index.index_folder(&wanted)?; let options = SearchOptions { folder: Some(wanted.clone()), extension: Some("md".into()), ..Default::default() }; assert_eq!( index.search_with_options("shared", 1, &options)?[0].filename, "shared_last.md" ); assert_eq!( index.search_with_options("retrieval", 1, &options)?[0].filename, "shared_last.md" ); assert_eq!( index.search_with_options("ha", 1, &options)?[0].filename, "shared_last.md" ); let result = index.search_with_options("unique-evidence", 1, &options)?; assert!(result[0].source.is_some()); assert!(result[0].snippet.contains("unique-evidence")); Ok(()) } #[test] fn exact_priority_literal_identifiers_and_index_lifecycle() -> Result<()> { let root = tempfile::tempdir()?; fs::create_dir(root.path().join("contracts"))?; fs::write( root.path().join("name_[%].txt"), "identifier ABC-123 and term", )?; fs::write(root.path().join("name_[%].txt.backup"), "term")?; fs::write( root.path().join("contracts").join("other.md"), "name identifier ABC-123", )?; fs::write(root.path().join("annual_report.md"), "unrelated text")?; fs::write(root.path().join("abc_missing_xyz.txt"), "unrelated text")?; let mut index = SearchIndex::open(":memory:")?; index.index_folder(root.path())?; let result = index.search("NAME_[%].TXT", 10)?; assert_eq!(result[0].match_kind, "exact_filename"); assert_eq!(result[1].match_kind, "filename"); assert_eq!(index.search("contracts", 1)?[0].match_kind, "path"); assert_eq!(index.search("ABC-123", 10)?.len(), 2); assert_eq!( index.search("annual report", 1)?[0].filename, "annual_report.md" ); assert!(index.search("abc_xyz", 10)?.is_empty()); let first = index .search("name", 10)? .into_iter() .map(|row| row.path) .collect::>(); let second = index .search("name", 10)? .into_iter() .map(|row| row.path) .collect::>(); assert_eq!(first, second); fs::rename( root.path().join("name_[%].txt"), root.path().join("replacement.txt"), )?; fs::remove_file(root.path().join("name_[%].txt.backup"))?; index.index_folder(root.path())?; assert!(index.search("name_[%].txt", 10)?.is_empty()); assert_eq!(index.search("placement", 1)?[0].filename, "replacement.txt"); Ok(()) } #[test] fn v4_migration_backfills_names_and_preserves_passage_ids() -> Result<()> { let root = tempfile::tempdir()?; let db = root.path().join("old.db"); let connection = rusqlite::Connection::open(&db)?; connection.execute_batch("CREATE TABLE files(id INTEGER PRIMARY KEY,path TEXT NOT NULL UNIQUE, root TEXT NOT NULL,filename TEXT NOT NULL,content TEXT NOT NULL,size INTEGER NOT NULL,mtime INTEGER NOT NULL, extension TEXT NOT NULL,extraction_error TEXT); CREATE VIRTUAL TABLE search USING fts5(filename,path,content,content='files',content_rowid='id',tokenize='unicode61'); CREATE VIRTUAL TABLE passages USING fts5(text,file_path UNINDEXED,kind UNINDEXED,number UNINDEXED); INSERT INTO files VALUES(41,'/fixture/report.pdf','/fixture','report.pdf','preserved evidence',18,123,'pdf',NULL); INSERT INTO search(search) VALUES('rebuild'); INSERT INTO passages(rowid,text,file_path,kind,number) VALUES(88,'preserved evidence','/fixture/report.pdf','page',7); PRAGMA user_version=4;")?; drop(connection); let index = SearchIndex::open(&db)?; assert_eq!(index.search("port.p", 1)?[0].filename, "report.pdf"); assert_eq!( index.search("evidence", 1)?[0] .source .as_ref() .unwrap() .number, 7 ); assert_eq!(index.passage_batch(0, 1)?[0].id, 88); Ok(()) } #[test] fn background_statistics_preserve_results_across_reopen_and_changes() -> Result<()> { let root = tempfile::tempdir()?; let db = root.path().join("index.db"); fs::write(root.path().join("invoice_1042.txt"), "billing evidence")?; let mut index = SearchIndex::open(&db)?; index.index_folder(root.path())?; assert!(index.prewarm()? > 0); drop(index); let mut index = SearchIndex::open(&db)?; assert_eq!(index.search("1042", 1)?[0].filename, "invoice_1042.txt"); fs::rename( root.path().join("invoice_1042.txt"), root.path().join("new_pattern.txt"), )?; index.index_folder(root.path())?; // Stale statistics affect only candidate cost, never substring correctness. assert_eq!(index.search("pattern", 1)?[0].filename, "new_pattern.txt"); assert!(index.search("1042", 1)?.is_empty()); index.prewarm()?; assert_eq!(index.search("pattern", 1)?[0].filename, "new_pattern.txt"); Ok(()) } #[test] fn positional_candidates_preserve_repeated_identifiers_unicode_and_path_matches() -> Result<()> { let root = tempfile::tempdir()?; let nested = root.path().join("folder_0000001_rare"); fs::create_dir(&nested)?; for (name, text) in [ ("ζαζα_0000001_rare.txt", "first source"), ("ζαζα_0000101_rare.txt", "second source"), ("backup_ζαζα_0000001_rare.txt", "backup source"), ] { fs::write(root.path().join(name), text)?; } fs::write(nested.join("other.md"), "path source")?; let mut index = SearchIndex::open(":memory:")?; index.index_folder(root.path())?; index.prewarm()?; let matches = index.search("0000001_rare", 20)?; assert_eq!(matches.len(), 3); assert_eq!(matches[0].match_kind, "filename"); assert_eq!(matches[1].match_kind, "filename"); assert_eq!(matches[2].match_kind, "path"); assert_eq!( index.search("ζαζα_0000101_rare", 1)?[0].filename, "ζαζα_0000101_rare.txt" ); let filtered = SearchOptions { folder: Some(nested), extension: Some("md".into()), ..Default::default() }; assert_eq!( index.search_with_options("0000001_rare", 1, &filtered)?[0].filename, "other.md" ); assert!(index.search("0000010_rare", 20)?.is_empty()); Ok(()) } #[test] fn candidate_first_content_matches_complete_filtered_ranking_and_sources() -> Result<()> { let root = tempfile::tempdir()?; fs::write(root.path().join("needle.txt"), "needle once")?; for number in 0..50 { let text = format!( "{} background evidence {}", "needle ".repeat(number % 7 + 1), "filler ".repeat(number % 5) ); fs::write(root.path().join(format!("notes_{number:03}.md")), text)?; } let mut index = SearchIndex::open(":memory:")?; index.index_folder(root.path())?; index.prewarm()?; let fast = index.search("needle", 20)?; // This nonrestrictive filter selects the normal complete filtered lane. let filtered = index.search_with_options( "needle", 20, &SearchOptions { min_size: Some(0), ..Default::default() }, )?; assert_eq!(fast.len(), 20); assert_eq!(filtered.len(), 20); for (fast, filtered) in fast.iter().zip(&filtered) { assert_eq!(fast.path, filtered.path); assert_eq!(fast.match_kind, filtered.match_kind); assert!((fast.score - filtered.score).abs() < 1e-12); assert_eq!(fast.snippet, filtered.snippet); assert_eq!( fast.source.as_ref().unwrap().number, filtered.source.as_ref().unwrap().number ); } assert_eq!(fast[0].match_kind, "filename"); Ok(()) }