Watch selected folders and reconcile bounded changes with media freshness guards

crates/searchi-core/tests/index_options.rs

@@ -12,6 +12,7 @@ fn exclusions_database_and_unchanged_metadata() -> Result<()> {
     fs::write(root.path().join("visible.txt"), "original content")?;
     let mut index = SearchIndex::open(root.path().join("index.db"))?;
     let options = IndexOptions {
+        excluded_paths: Vec::new(),
         excluded_directory_names: vec![
             ".git".into(),
             "node_modules".into(),

crates/searchi-core/tests/targeted_reconciliation.rs

@@ -0,0 +1,484 @@
+//! Atomic bounded event reconciliation, independent of platform watcher delivery.
+use anyhow::Result;
+use searchi_core::{IndexOptions, SearchIndex};
+use std::{fs, path::PathBuf, sync::atomic::AtomicBool};
+
+#[test]
+fn same_stat_overwrite_and_unrelated_files_are_preserved() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    let file = root.join("changed.txt");
+    fs::write(&file, "oldword alpha")?;
+    fs::write(root.join("other.txt"), "unrelated beta")?;
+    let root = fs::canonicalize(root)?;
+    let file = fs::canonicalize(file)?;
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    index.index_folder(&root)?;
+    let mtime = fs::metadata(&file)?.modified()?;
+    fs::write(&file, "newword alpha")?;
+    fs::File::options()
+        .write(true)
+        .open(&file)?
+        .set_times(fs::FileTimes::new().set_modified(mtime))?;
+    let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
+    assert_eq!(report.discovered, 1);
+    assert_eq!(report.indexed, 1);
+    assert_eq!(index.search("newword", 10)?.len(), 1);
+    assert!(index.search("oldword", 10)?.is_empty());
+    assert_eq!(index.search("unrelated", 10)?.len(), 1);
+    Ok(())
+}
+
+#[test]
+fn rename_subtree_deletion_and_literal_boundaries_preserve_other_roots() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    let child = root.join("a_%");
+    fs::create_dir(&child)?;
+    let sibling = root.join("a_%suffix");
+    fs::create_dir(&sibling)?;
+    fs::write(child.join("old.txt"), "firstrecord")?;
+    fs::write(sibling.join("keep.txt"), "siblingrecord")?;
+    let other = temp.path().join("other");
+    fs::create_dir(&other)?;
+    fs::write(other.join("keep.txt"), "otherrootrecord")?;
+    let root = fs::canonicalize(root)?;
+    let child = root.join("a_%");
+    let old = child.join("old.txt");
+    let new = child.join("renamed.txt");
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    index.index_folder(&root)?;
+    index.index_folder(&other)?;
+    fs::rename(&old, &new)?;
+    let report = index.index_paths_with_options(
+        &root,
+        &[old, new.clone(), new],
+        &IndexOptions::default(),
+    )?;
+    assert_eq!(report.indexed, 1);
+    assert_eq!(report.removed, 1);
+    assert_eq!(index.search("firstrecord", 10)?[0].filename, "renamed.txt");
+    fs::remove_dir_all(&child)?;
+    let report = index.index_paths_with_options(&root, &[child], &IndexOptions::default())?;
+    assert_eq!(report.removed, 1);
+    assert_eq!(index.search("siblingrecord", 10)?.len(), 1);
+    assert_eq!(index.search("otherrootrecord", 10)?.len(), 1);
+    Ok(())
+}
+
+#[test]
+fn directory_events_reconcile_only_subtree_and_prune_exclusions() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::create_dir(root.join("nested"))?;
+    fs::create_dir(root.join("target"))?;
+    fs::write(root.join("nested/one.txt"), "oldnested")?;
+    fs::write(root.join("top.txt"), "toporiginal")?;
+    let root = fs::canonicalize(root)?;
+    let mut index = SearchIndex::open(root.join("index.db"))?;
+    index.index_folder(&root)?;
+    fs::remove_file(root.join("nested/one.txt"))?;
+    fs::write(root.join("nested/two.txt"), "newnested")?;
+    fs::write(root.join("top.txt"), "topchanged")?;
+    fs::write(root.join("target/excluded.txt"), "excludedcontent")?;
+    let report = index.index_paths_with_options(
+        &root,
+        &[
+            root.join("nested"),
+            root.join("nested/two.txt"),
+            root.join("target"),
+        ],
+        &IndexOptions::default(),
+    )?;
+    assert_eq!(report.discovered, 1);
+    assert_eq!(report.removed, 1);
+    assert_eq!(index.search("newnested", 10)?.len(), 1);
+    assert_eq!(index.search("toporiginal", 10)?.len(), 1);
+    assert!(index.search("topchanged", 10)?.is_empty());
+    assert!(index.search("excludedcontent", 10)?.is_empty());
+    assert!(index.search("index.db", 10)?.is_empty());
+    Ok(())
+}
+
+#[test]
+fn invalid_batches_and_failed_reads_roll_back_all_changes() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::write(root.join("a.txt"), "beforetransaction")?;
+    fs::write(root.join("z.txt"), "validlast")?;
+    let root = fs::canonicalize(root)?;
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    index.index_folder(&root)?;
+    fs::write(root.join("a.txt"), "aftertransaction")?;
+    #[cfg(windows)]
+    let locked = {
+        use std::os::windows::fs::OpenOptionsExt;
+        fs::OpenOptions::new()
+            .read(true)
+            .share_mode(0)
+            .open(root.join("z.txt"))?
+    };
+    #[cfg(unix)]
+    let permissions = {
+        use std::os::unix::fs::PermissionsExt;
+        let previous = fs::metadata(root.join("z.txt"))?.permissions();
+        fs::set_permissions(root.join("z.txt"), fs::Permissions::from_mode(0))?;
+        previous
+    };
+    let failed = index.index_paths_with_options(
+        &root,
+        &[root.join("a.txt"), root.join("z.txt")],
+        &IndexOptions::default(),
+    );
+    #[cfg(windows)]
+    drop(locked);
+    #[cfg(unix)]
+    fs::set_permissions(root.join("z.txt"), permissions)?;
+    assert!(failed.is_err());
+    assert_eq!(index.search("beforetransaction", 10)?.len(), 1);
+    assert!(index.search("aftertransaction", 10)?.is_empty());
+    assert!(index
+        .index_paths_with_options(
+            &root,
+            &[temp.path().join("outside.txt")],
+            &IndexOptions::default()
+        )
+        .is_err());
+    assert!(index
+        .index_paths_with_options(
+            &root,
+            &[PathBuf::from("relative.txt")],
+            &IndexOptions::default()
+        )
+        .is_err());
+    assert!(index
+        .index_paths_with_options(
+            &root,
+            &[root.join("../outside.txt")],
+            &IndexOptions::default()
+        )
+        .is_err());
+    assert!(index
+        .index_paths_with_options(
+            &root,
+            &vec![root.join("a.txt"); 129],
+            &IndexOptions::default()
+        )
+        .is_err());
+    Ok(())
+}
+
+#[test]
+fn cancellation_and_unavailable_root_preserve_records() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::write(root.join("file.txt"), "preservedrecord")?;
+    let root = fs::canonicalize(root)?;
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    index.index_folder(&root)?;
+    fs::remove_file(root.join("file.txt"))?;
+    let cancelled = AtomicBool::new(true);
+    assert!(index
+        .index_paths_cancellable(
+            &root,
+            &[root.join("file.txt")],
+            &IndexOptions::default(),
+            &cancelled
+        )
+        .is_err());
+    assert!(index
+        .index_folder_cancellable(&root, &IndexOptions::default(), &cancelled)
+        .is_err());
+    assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
+    fs::remove_dir(&root)?;
+    assert!(index
+        .index_paths_with_options(&root, &[root.join("file.txt")], &IndexOptions::default())
+        .is_err());
+    assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
+    Ok(())
+}
+
+#[cfg(unix)]
+#[test]
+fn symlink_ancestor_cannot_index_outside_content() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    let outside = temp.path().join("outside");
+    fs::create_dir(&outside)?;
+    fs::write(outside.join("secret.txt"), "secretoutside")?;
+    let root = fs::canonicalize(root)?;
+    std::os::unix::fs::symlink(&outside, root.join("link"))?;
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    assert!(index
+        .index_paths_with_options(
+            &root,
+            &[root.join("link/secret.txt")],
+            &IndexOptions::default()
+        )
+        .is_err());
+    assert!(index.search("secretoutside", 10)?.is_empty());
+    Ok(())
+}
+
+#[test]
+fn explicit_exclusions_remove_prior_records_without_prefix_leakage() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::create_dir(root.join("cache"))?;
+    fs::create_dir(root.join("cache-extra"))?;
+    fs::write(root.join("cache/hidden.txt"), "previouslyvisible")?;
+    fs::write(root.join("cache-extra/keep.txt"), "preservedneighbor")?;
+    let root = fs::canonicalize(root)?;
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    index.index_folder(&root)?;
+    let mut options = IndexOptions::default();
+    options.excluded_paths.push(root.join("cache"));
+    let report = index.index_paths_with_options(&root, &[root.join("cache")], &options)?;
+    assert_eq!(report.removed, 1);
+    assert!(index.search("previouslyvisible", 10)?.is_empty());
+    assert_eq!(index.search("preservedneighbor", 10)?.len(), 1);
+    assert_eq!(
+        index.index_folder_with_options(&root, &options)?.discovered,
+        1
+    );
+    options.excluded_paths.push(PathBuf::from("relative"));
+    assert!(index.index_folder_with_options(&root, &options).is_err());
+    Ok(())
+}
+
+#[test]
+fn independent_reader_remains_available_and_sees_atomic_update() -> Result<()> {
+    use std::sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    };
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    for number in 0..64 {
+        fs::write(root.join(format!("{number}.txt")), "oldversion")?;
+    }
+    let root = fs::canonicalize(root)?;
+    let db = temp.path().join("index.db");
+    let mut writer = SearchIndex::open(&db)?;
+    writer.index_folder(&root)?;
+    let reader = SearchIndex::open(&db)?;
+    for number in 0..64 {
+        fs::write(
+            root.join(format!("{number}.txt")),
+            format!("newversion {}", "content ".repeat(4000)),
+        )?;
+    }
+    let finished = Arc::new(AtomicBool::new(false));
+    let done = finished.clone();
+    std::thread::scope(|scope| -> Result<()> {
+        let task = scope.spawn(move || {
+            let result = writer.index_paths_with_options(
+                &root,
+                std::slice::from_ref(&root),
+                &IndexOptions::default(),
+            );
+            done.store(true, Ordering::Release);
+            result
+        });
+        let mut observations = 0;
+        while !finished.load(Ordering::Acquire) {
+            let count = reader.search("oldversion", 100)?.len();
+            assert!(
+                count == 64 || count == 0,
+                "Reader must never see partially committed records"
+            );
+            observations += 1;
+        }
+        task.join().unwrap()?;
+        assert!(observations > 0);
+        assert_eq!(reader.search("newversion", 100)?.len(), 64);
+        Ok(())
+    })?;
+    Ok(())
+}
+
+#[test]
+fn cancellation_after_write_transaction_begins_rolls_back_batch() -> Result<()> {
+    use std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        time::{Duration, Instant},
+    };
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    for number in 0..128 {
+        fs::write(root.join(format!("{number}.txt")), "oldcancelrecord")?;
+    }
+    let root = fs::canonicalize(root)?;
+    let db = temp.path().join("index.db");
+    let mut writer = SearchIndex::open(&db)?;
+    writer.index_folder(&root)?;
+    let reader = SearchIndex::open(&db)?;
+    for number in 0..128 {
+        fs::write(
+            root.join(format!("{number}.txt")),
+            format!("newcancelrecord {}", "new content ".repeat(8000)),
+        )?;
+    }
+    let probe = rusqlite::Connection::open(&db)?;
+    probe.busy_timeout(Duration::ZERO)?;
+    let cancelled = Arc::new(AtomicBool::new(false));
+    let flag = cancelled.clone();
+    std::thread::scope(|scope| -> Result<()> {
+        let indexing = scope
+            .spawn(move || writer.index_folder_cancellable(&root, &IndexOptions::default(), &flag));
+        let deadline = Instant::now() + Duration::from_secs(5);
+        let mut observed_write = false;
+        while Instant::now() < deadline {
+            match probe.execute_batch("BEGIN IMMEDIATE") {
+                Ok(()) => probe.execute_batch("ROLLBACK")?,
+                Err(rusqlite::Error::SqliteFailure(error, _))
+                    if error.code == rusqlite::ErrorCode::DatabaseBusy =>
+                {
+                    observed_write = true;
+                    cancelled.store(true, Ordering::Release);
+                    break;
+                }
+                Err(error) => return Err(error.into()),
+            }
+            std::thread::sleep(Duration::from_millis(1));
+        }
+        let result = indexing.join().unwrap();
+        assert!(
+            observed_write,
+            "Must observe the writer transaction before cancellation"
+        );
+        assert!(result.is_err());
+        assert_eq!(reader.search("oldcancelrecord", 200)?.len(), 128);
+        assert!(reader.search("newcancelrecord", 200)?.is_empty());
+        Ok(())
+    })?;
+    Ok(())
+}
+
+#[test]
+fn same_stat_media_hint_advances_revision_and_removes_only_its_tags() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::write(root.join("changed.wav"), [1u8, 2, 3, 4])?;
+    fs::write(root.join("unrelated.wav"), [5u8, 6, 7, 8])?;
+    let root = fs::canonicalize(root)?;
+    let file = root.join("changed.wav");
+    let db = temp.path().join("index.db");
+    let mut index = SearchIndex::open(&db)?;
+    index.index_folder(&root)?;
+    let path = file.to_string_lossy().into_owned();
+    let (size, mtime) = index.file_signature(&path)?.unwrap();
+    index.set_media_metadata(&path, "stale artist metadata", 1000, size, mtime)?;
+    let original_mtime = fs::metadata(&file)?.modified()?;
+    fs::write(&file, [4u8, 3, 2, 1])?;
+    fs::OpenOptions::new()
+        .write(true)
+        .open(&file)?
+        .set_times(fs::FileTimes::new().set_modified(original_mtime))?;
+    assert_eq!(
+        index.index_folder(&root)?.unchanged,
+        2,
+        "Periodic metadata scan retains fast path"
+    );
+    let connection = rusqlite::Connection::open(&db)?;
+    let revisions = || -> Result<Vec<(String, i64)>> {
+        let mut statement =
+            connection.prepare("SELECT filename,change_revision FROM files ORDER BY filename")?;
+        let rows = statement.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
+        Ok(rows.collect::<rusqlite::Result<_>>()?)
+    };
+    assert_eq!(
+        revisions()?,
+        vec![("changed.wav".into(), 0), ("unrelated.wav".into(), 0)]
+    );
+    assert_eq!(index.search("stale artist", 10)?.len(), 1);
+    let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
+    assert_eq!(report.indexed, 1);
+    assert_eq!(report.unchanged, 0);
+    assert_eq!(
+        revisions()?,
+        vec![("changed.wav".into(), 1), ("unrelated.wav".into(), 0)]
+    );
+    assert!(index.search("stale artist", 10)?.is_empty());
+    assert_eq!(index.search("changed.wav", 10)?.len(), 1);
+    let report = index.index_paths_with_options(
+        &root,
+        std::slice::from_ref(&root),
+        &IndexOptions::default(),
+    )?;
+    assert_eq!(report.indexed, 2);
+    assert_eq!(
+        revisions()?,
+        vec![("changed.wav".into(), 2), ("unrelated.wav".into(), 1)]
+    );
+    Ok(())
+}
+
+#[test]
+fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::write(root.join("document.txt"), "preserved migration text")?;
+    let db = temp.path().join("index.db");
+    let mut index = SearchIndex::open(&db)?;
+    index.index_folder(&root)?;
+    drop(index);
+    let connection = rusqlite::Connection::open(&db)?;
+    connection
+        .execute_batch("ALTER TABLE files DROP COLUMN change_revision; PRAGMA user_version=12;")?;
+    drop(connection);
+    let mut index = SearchIndex::open(&db)?;
+    assert_eq!(index.search("migration text", 10)?.len(), 1);
+    assert_eq!(index.index_folder(&root)?.unchanged, 1);
+    let connection = rusqlite::Connection::open(&db)?;
+    assert_eq!(
+        connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
+        13
+    );
+    assert_eq!(
+        connection.query_row("SELECT change_revision FROM files", [], |row| row
+            .get::<_, i64>(0))?,
+        0
+    );
+    Ok(())
+}
+
+#[test]
+fn metadata_probe_revision_rejects_tags_after_same_stat_hint() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("root");
+    fs::create_dir(&root)?;
+    fs::write(root.join("song.wav"), [1u8, 2, 3, 4])?;
+    let root = fs::canonicalize(root)?;
+    let file = root.join("song.wav");
+    let path = file.to_string_lossy().into_owned();
+    let mut index = SearchIndex::open(temp.path().join("index.db"))?;
+    index.index_folder(&root)?;
+    let (size, mtime) = index.file_signature(&path)?.unwrap();
+    let before = index.media_change_revision(&path, size, mtime)?;
+    index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
+    assert!(index
+        .set_media_metadata_at_revision(&path, "stale probe tags", 1000, size, mtime, before)
+        .is_err());
+    assert!(index.search("stale probe", 10)?.is_empty());
+    let current = index.media_change_revision(&path, size, mtime)?;
+    assert_eq!(current, before + 1);
+    index.set_media_metadata_at_revision(&path, "fresh probe tags", 1000, size, mtime, current)?;
+    assert_eq!(index.search("fresh probe", 10)?.len(), 1);
+    assert!(index.media_change_revision(&path, size + 1, mtime).is_err());
+    Ok(())
+}