485 lines
18 KiB
Rust
485 lines
18 KiB
Rust
//! Atomic bounded event reconciliation, independent of platform watcher delivery.
|
|
use anyhow::Result;
|
|
use searchi_core::{IndexOptions, SearchIndex};
|
|
use std::{fs, path::PathBuf, sync::atomic::AtomicBool};
|
|
|
|
#[test]
|
|
fn same_stat_overwrite_and_unrelated_files_are_preserved() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
let file = root.join("changed.txt");
|
|
fs::write(&file, "oldword alpha")?;
|
|
fs::write(root.join("other.txt"), "unrelated beta")?;
|
|
let root = fs::canonicalize(root)?;
|
|
let file = fs::canonicalize(file)?;
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
let mtime = fs::metadata(&file)?.modified()?;
|
|
fs::write(&file, "newword alpha")?;
|
|
fs::File::options()
|
|
.write(true)
|
|
.open(&file)?
|
|
.set_times(fs::FileTimes::new().set_modified(mtime))?;
|
|
let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
|
|
assert_eq!(report.discovered, 1);
|
|
assert_eq!(report.indexed, 1);
|
|
assert_eq!(index.search("newword", 10)?.len(), 1);
|
|
assert!(index.search("oldword", 10)?.is_empty());
|
|
assert_eq!(index.search("unrelated", 10)?.len(), 1);
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn rename_subtree_deletion_and_literal_boundaries_preserve_other_roots() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
let child = root.join("a_%");
|
|
fs::create_dir(&child)?;
|
|
let sibling = root.join("a_%suffix");
|
|
fs::create_dir(&sibling)?;
|
|
fs::write(child.join("old.txt"), "firstrecord")?;
|
|
fs::write(sibling.join("keep.txt"), "siblingrecord")?;
|
|
let other = temp.path().join("other");
|
|
fs::create_dir(&other)?;
|
|
fs::write(other.join("keep.txt"), "otherrootrecord")?;
|
|
let root = fs::canonicalize(root)?;
|
|
let child = root.join("a_%");
|
|
let old = child.join("old.txt");
|
|
let new = child.join("renamed.txt");
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
index.index_folder(&other)?;
|
|
fs::rename(&old, &new)?;
|
|
let report = index.index_paths_with_options(
|
|
&root,
|
|
&[old, new.clone(), new],
|
|
&IndexOptions::default(),
|
|
)?;
|
|
assert_eq!(report.indexed, 1);
|
|
assert_eq!(report.removed, 1);
|
|
assert_eq!(index.search("firstrecord", 10)?[0].filename, "renamed.txt");
|
|
fs::remove_dir_all(&child)?;
|
|
let report = index.index_paths_with_options(&root, &[child], &IndexOptions::default())?;
|
|
assert_eq!(report.removed, 1);
|
|
assert_eq!(index.search("siblingrecord", 10)?.len(), 1);
|
|
assert_eq!(index.search("otherrootrecord", 10)?.len(), 1);
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn directory_events_reconcile_only_subtree_and_prune_exclusions() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::create_dir(root.join("nested"))?;
|
|
fs::create_dir(root.join("target"))?;
|
|
fs::write(root.join("nested/one.txt"), "oldnested")?;
|
|
fs::write(root.join("top.txt"), "toporiginal")?;
|
|
let root = fs::canonicalize(root)?;
|
|
let mut index = SearchIndex::open(root.join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
fs::remove_file(root.join("nested/one.txt"))?;
|
|
fs::write(root.join("nested/two.txt"), "newnested")?;
|
|
fs::write(root.join("top.txt"), "topchanged")?;
|
|
fs::write(root.join("target/excluded.txt"), "excludedcontent")?;
|
|
let report = index.index_paths_with_options(
|
|
&root,
|
|
&[
|
|
root.join("nested"),
|
|
root.join("nested/two.txt"),
|
|
root.join("target"),
|
|
],
|
|
&IndexOptions::default(),
|
|
)?;
|
|
assert_eq!(report.discovered, 1);
|
|
assert_eq!(report.removed, 1);
|
|
assert_eq!(index.search("newnested", 10)?.len(), 1);
|
|
assert_eq!(index.search("toporiginal", 10)?.len(), 1);
|
|
assert!(index.search("topchanged", 10)?.is_empty());
|
|
assert!(index.search("excludedcontent", 10)?.is_empty());
|
|
assert!(index.search("index.db", 10)?.is_empty());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_batches_and_failed_reads_roll_back_all_changes() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::write(root.join("a.txt"), "beforetransaction")?;
|
|
fs::write(root.join("z.txt"), "validlast")?;
|
|
let root = fs::canonicalize(root)?;
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
fs::write(root.join("a.txt"), "aftertransaction")?;
|
|
#[cfg(windows)]
|
|
let locked = {
|
|
use std::os::windows::fs::OpenOptionsExt;
|
|
fs::OpenOptions::new()
|
|
.read(true)
|
|
.share_mode(0)
|
|
.open(root.join("z.txt"))?
|
|
};
|
|
#[cfg(unix)]
|
|
let permissions = {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let previous = fs::metadata(root.join("z.txt"))?.permissions();
|
|
fs::set_permissions(root.join("z.txt"), fs::Permissions::from_mode(0))?;
|
|
previous
|
|
};
|
|
let failed = index.index_paths_with_options(
|
|
&root,
|
|
&[root.join("a.txt"), root.join("z.txt")],
|
|
&IndexOptions::default(),
|
|
);
|
|
#[cfg(windows)]
|
|
drop(locked);
|
|
#[cfg(unix)]
|
|
fs::set_permissions(root.join("z.txt"), permissions)?;
|
|
assert!(failed.is_err());
|
|
assert_eq!(index.search("beforetransaction", 10)?.len(), 1);
|
|
assert!(index.search("aftertransaction", 10)?.is_empty());
|
|
assert!(index
|
|
.index_paths_with_options(
|
|
&root,
|
|
&[temp.path().join("outside.txt")],
|
|
&IndexOptions::default()
|
|
)
|
|
.is_err());
|
|
assert!(index
|
|
.index_paths_with_options(
|
|
&root,
|
|
&[PathBuf::from("relative.txt")],
|
|
&IndexOptions::default()
|
|
)
|
|
.is_err());
|
|
assert!(index
|
|
.index_paths_with_options(
|
|
&root,
|
|
&[root.join("../outside.txt")],
|
|
&IndexOptions::default()
|
|
)
|
|
.is_err());
|
|
assert!(index
|
|
.index_paths_with_options(
|
|
&root,
|
|
&vec![root.join("a.txt"); 129],
|
|
&IndexOptions::default()
|
|
)
|
|
.is_err());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn cancellation_and_unavailable_root_preserve_records() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::write(root.join("file.txt"), "preservedrecord")?;
|
|
let root = fs::canonicalize(root)?;
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
fs::remove_file(root.join("file.txt"))?;
|
|
let cancelled = AtomicBool::new(true);
|
|
assert!(index
|
|
.index_paths_cancellable(
|
|
&root,
|
|
&[root.join("file.txt")],
|
|
&IndexOptions::default(),
|
|
&cancelled
|
|
)
|
|
.is_err());
|
|
assert!(index
|
|
.index_folder_cancellable(&root, &IndexOptions::default(), &cancelled)
|
|
.is_err());
|
|
assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
|
|
fs::remove_dir(&root)?;
|
|
assert!(index
|
|
.index_paths_with_options(&root, &[root.join("file.txt")], &IndexOptions::default())
|
|
.is_err());
|
|
assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[test]
|
|
fn symlink_ancestor_cannot_index_outside_content() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
let outside = temp.path().join("outside");
|
|
fs::create_dir(&outside)?;
|
|
fs::write(outside.join("secret.txt"), "secretoutside")?;
|
|
let root = fs::canonicalize(root)?;
|
|
std::os::unix::fs::symlink(&outside, root.join("link"))?;
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
assert!(index
|
|
.index_paths_with_options(
|
|
&root,
|
|
&[root.join("link/secret.txt")],
|
|
&IndexOptions::default()
|
|
)
|
|
.is_err());
|
|
assert!(index.search("secretoutside", 10)?.is_empty());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_exclusions_remove_prior_records_without_prefix_leakage() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::create_dir(root.join("cache"))?;
|
|
fs::create_dir(root.join("cache-extra"))?;
|
|
fs::write(root.join("cache/hidden.txt"), "previouslyvisible")?;
|
|
fs::write(root.join("cache-extra/keep.txt"), "preservedneighbor")?;
|
|
let root = fs::canonicalize(root)?;
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
let mut options = IndexOptions::default();
|
|
options.excluded_paths.push(root.join("cache"));
|
|
let report = index.index_paths_with_options(&root, &[root.join("cache")], &options)?;
|
|
assert_eq!(report.removed, 1);
|
|
assert!(index.search("previouslyvisible", 10)?.is_empty());
|
|
assert_eq!(index.search("preservedneighbor", 10)?.len(), 1);
|
|
assert_eq!(
|
|
index.index_folder_with_options(&root, &options)?.discovered,
|
|
1
|
|
);
|
|
options.excluded_paths.push(PathBuf::from("relative"));
|
|
assert!(index.index_folder_with_options(&root, &options).is_err());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn independent_reader_remains_available_and_sees_atomic_update() -> Result<()> {
|
|
use std::sync::{
|
|
atomic::{AtomicBool, Ordering},
|
|
Arc,
|
|
};
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
for number in 0..64 {
|
|
fs::write(root.join(format!("{number}.txt")), "oldversion")?;
|
|
}
|
|
let root = fs::canonicalize(root)?;
|
|
let db = temp.path().join("index.db");
|
|
let mut writer = SearchIndex::open(&db)?;
|
|
writer.index_folder(&root)?;
|
|
let reader = SearchIndex::open(&db)?;
|
|
for number in 0..64 {
|
|
fs::write(
|
|
root.join(format!("{number}.txt")),
|
|
format!("newversion {}", "content ".repeat(4000)),
|
|
)?;
|
|
}
|
|
let finished = Arc::new(AtomicBool::new(false));
|
|
let done = finished.clone();
|
|
std::thread::scope(|scope| -> Result<()> {
|
|
let task = scope.spawn(move || {
|
|
let result = writer.index_paths_with_options(
|
|
&root,
|
|
std::slice::from_ref(&root),
|
|
&IndexOptions::default(),
|
|
);
|
|
done.store(true, Ordering::Release);
|
|
result
|
|
});
|
|
let mut observations = 0;
|
|
while !finished.load(Ordering::Acquire) {
|
|
let count = reader.search("oldversion", 100)?.len();
|
|
assert!(
|
|
count == 64 || count == 0,
|
|
"Reader must never see partially committed records"
|
|
);
|
|
observations += 1;
|
|
}
|
|
task.join().unwrap()?;
|
|
assert!(observations > 0);
|
|
assert_eq!(reader.search("newversion", 100)?.len(), 64);
|
|
Ok(())
|
|
})?;
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn cancellation_after_write_transaction_begins_rolls_back_batch() -> Result<()> {
|
|
use std::{
|
|
sync::{
|
|
atomic::{AtomicBool, Ordering},
|
|
Arc,
|
|
},
|
|
time::{Duration, Instant},
|
|
};
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
for number in 0..128 {
|
|
fs::write(root.join(format!("{number}.txt")), "oldcancelrecord")?;
|
|
}
|
|
let root = fs::canonicalize(root)?;
|
|
let db = temp.path().join("index.db");
|
|
let mut writer = SearchIndex::open(&db)?;
|
|
writer.index_folder(&root)?;
|
|
let reader = SearchIndex::open(&db)?;
|
|
for number in 0..128 {
|
|
fs::write(
|
|
root.join(format!("{number}.txt")),
|
|
format!("newcancelrecord {}", "new content ".repeat(8000)),
|
|
)?;
|
|
}
|
|
let probe = rusqlite::Connection::open(&db)?;
|
|
probe.busy_timeout(Duration::ZERO)?;
|
|
let cancelled = Arc::new(AtomicBool::new(false));
|
|
let flag = cancelled.clone();
|
|
std::thread::scope(|scope| -> Result<()> {
|
|
let indexing = scope
|
|
.spawn(move || writer.index_folder_cancellable(&root, &IndexOptions::default(), &flag));
|
|
let deadline = Instant::now() + Duration::from_secs(5);
|
|
let mut observed_write = false;
|
|
while Instant::now() < deadline {
|
|
match probe.execute_batch("BEGIN IMMEDIATE") {
|
|
Ok(()) => probe.execute_batch("ROLLBACK")?,
|
|
Err(rusqlite::Error::SqliteFailure(error, _))
|
|
if error.code == rusqlite::ErrorCode::DatabaseBusy =>
|
|
{
|
|
observed_write = true;
|
|
cancelled.store(true, Ordering::Release);
|
|
break;
|
|
}
|
|
Err(error) => return Err(error.into()),
|
|
}
|
|
std::thread::sleep(Duration::from_millis(1));
|
|
}
|
|
let result = indexing.join().unwrap();
|
|
assert!(
|
|
observed_write,
|
|
"Must observe the writer transaction before cancellation"
|
|
);
|
|
assert!(result.is_err());
|
|
assert_eq!(reader.search("oldcancelrecord", 200)?.len(), 128);
|
|
assert!(reader.search("newcancelrecord", 200)?.is_empty());
|
|
Ok(())
|
|
})?;
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn same_stat_media_hint_advances_revision_and_removes_only_its_tags() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::write(root.join("changed.wav"), [1u8, 2, 3, 4])?;
|
|
fs::write(root.join("unrelated.wav"), [5u8, 6, 7, 8])?;
|
|
let root = fs::canonicalize(root)?;
|
|
let file = root.join("changed.wav");
|
|
let db = temp.path().join("index.db");
|
|
let mut index = SearchIndex::open(&db)?;
|
|
index.index_folder(&root)?;
|
|
let path = file.to_string_lossy().into_owned();
|
|
let (size, mtime) = index.file_signature(&path)?.unwrap();
|
|
index.set_media_metadata(&path, "stale artist metadata", 1000, size, mtime)?;
|
|
let original_mtime = fs::metadata(&file)?.modified()?;
|
|
fs::write(&file, [4u8, 3, 2, 1])?;
|
|
fs::OpenOptions::new()
|
|
.write(true)
|
|
.open(&file)?
|
|
.set_times(fs::FileTimes::new().set_modified(original_mtime))?;
|
|
assert_eq!(
|
|
index.index_folder(&root)?.unchanged,
|
|
2,
|
|
"Periodic metadata scan retains fast path"
|
|
);
|
|
let connection = rusqlite::Connection::open(&db)?;
|
|
let revisions = || -> Result<Vec<(String, i64)>> {
|
|
let mut statement =
|
|
connection.prepare("SELECT filename,change_revision FROM files ORDER BY filename")?;
|
|
let rows = statement.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
|
|
Ok(rows.collect::<rusqlite::Result<_>>()?)
|
|
};
|
|
assert_eq!(
|
|
revisions()?,
|
|
vec![("changed.wav".into(), 0), ("unrelated.wav".into(), 0)]
|
|
);
|
|
assert_eq!(index.search("stale artist", 10)?.len(), 1);
|
|
let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
|
|
assert_eq!(report.indexed, 1);
|
|
assert_eq!(report.unchanged, 0);
|
|
assert_eq!(
|
|
revisions()?,
|
|
vec![("changed.wav".into(), 1), ("unrelated.wav".into(), 0)]
|
|
);
|
|
assert!(index.search("stale artist", 10)?.is_empty());
|
|
assert_eq!(index.search("changed.wav", 10)?.len(), 1);
|
|
let report = index.index_paths_with_options(
|
|
&root,
|
|
std::slice::from_ref(&root),
|
|
&IndexOptions::default(),
|
|
)?;
|
|
assert_eq!(report.indexed, 2);
|
|
assert_eq!(
|
|
revisions()?,
|
|
vec![("changed.wav".into(), 2), ("unrelated.wav".into(), 1)]
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::write(root.join("document.txt"), "preserved migration text")?;
|
|
let db = temp.path().join("index.db");
|
|
let mut index = SearchIndex::open(&db)?;
|
|
index.index_folder(&root)?;
|
|
drop(index);
|
|
let connection = rusqlite::Connection::open(&db)?;
|
|
connection
|
|
.execute_batch("ALTER TABLE files DROP COLUMN change_revision; PRAGMA user_version=12;")?;
|
|
drop(connection);
|
|
let mut index = SearchIndex::open(&db)?;
|
|
assert_eq!(index.search("migration text", 10)?.len(), 1);
|
|
assert_eq!(index.index_folder(&root)?.unchanged, 1);
|
|
let connection = rusqlite::Connection::open(&db)?;
|
|
assert_eq!(
|
|
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
|
|
14
|
|
);
|
|
assert_eq!(
|
|
connection.query_row("SELECT change_revision FROM files", [], |row| row
|
|
.get::<_, i64>(0))?,
|
|
0
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn metadata_probe_revision_rejects_tags_after_same_stat_hint() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let root = temp.path().join("root");
|
|
fs::create_dir(&root)?;
|
|
fs::write(root.join("song.wav"), [1u8, 2, 3, 4])?;
|
|
let root = fs::canonicalize(root)?;
|
|
let file = root.join("song.wav");
|
|
let path = file.to_string_lossy().into_owned();
|
|
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
|
|
index.index_folder(&root)?;
|
|
let (size, mtime) = index.file_signature(&path)?.unwrap();
|
|
let before = index.media_change_revision(&path, size, mtime)?;
|
|
index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
|
|
assert!(index
|
|
.set_media_metadata_at_revision(&path, "stale probe tags", 1000, size, mtime, before)
|
|
.is_err());
|
|
assert!(index.search("stale probe", 10)?.is_empty());
|
|
let current = index.media_change_revision(&path, size, mtime)?;
|
|
assert_eq!(current, before + 1);
|
|
index.set_media_metadata_at_revision(&path, "fresh probe tags", 1000, size, mtime, current)?;
|
|
assert_eq!(index.search("fresh probe", 10)?.len(), 1);
|
|
assert!(index.media_change_revision(&path, size + 1, mtime).is_err());
|
|
Ok(())
|
|
}
|