karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Watch selected folders and reconcile bounded changes with media freshness guards

Browse Source
This commit is contained in:
karl committed 2026-10-13 02:30:38 +02:00
1 parent 99748bd99f
commit 657de3de86
22 files changed
+2022 -149
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/searchi-core/Cargo.toml
+1
View File
@@ -5,6 +5,7 @@ edition.workspace = true
license.workspace = true
[dependencies]
searchi-media = { path = "../searchi-media" }
anyhow.workspace = true
serde.workspace = true
sha2 = "0.10"
crates/searchi-core/src/lib.rs
+359 -125
View File
@@ -17,11 +17,12 @@ use std::{
collections::{BTreeSet, HashMap},
fs,
path::{Path, PathBuf},
sync::atomic::{AtomicBool, Ordering},
time::UNIX_EPOCH,
};
use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 12;
const SCHEMA_VERSION: i64 = 13;
const MAX_RESULTS: usize = 1000;
#[derive(Debug, Default, Serialize)]
@@ -38,12 +39,15 @@ pub struct IndexReport {
#[derive(Debug, Clone)]
pub struct IndexOptions {
pub excluded_directory_names: Vec<String>,
/// Absolute component-normalized directory/file paths, matched by boundary.
pub excluded_paths: Vec<PathBuf>,
}
impl Default for IndexOptions {
fn default() -> Self {
Self {
excluded_directory_names: vec![".git".into(), "node_modules".into(), "target".into()],
excluded_paths: Vec::new(),
}
}
}
@@ -262,7 +266,17 @@ impl SearchIndex {
if !has_extraction_hash {
initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?;
}
initialization.execute_batch("PRAGMA user_version=12;")?;
let has_change_revision: bool = initialization.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='change_revision')",
[], |row| row.get(0),
)?;
if !has_change_revision {
initialization.execute(
"ALTER TABLE files ADD COLUMN change_revision INTEGER NOT NULL DEFAULT 0",
[],
)?;
}
initialization.execute_batch("PRAGMA user_version=13;")?;
initialization.commit()?;
}
Ok(Self {
@@ -295,6 +309,34 @@ impl SearchIndex {
duration_ms: u32,
size: u64,
mtime_ns: i64,
) -> Result<()> {
let revision = self.media_change_revision(path, size, mtime_ns)?;
self.set_media_metadata_at_revision(path, tags, duration_ms, size, mtime_ns, revision)
}
/// Capture before probing media. Event hints invalidate same-stat inputs
/// independently of their size and timestamp.
pub fn media_change_revision(&self, path: &str, size: u64, mtime_ns: i64) -> Result<i64> {
self.connection
.query_row(
"SELECT change_revision FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
|row| row.get(0),
)
.optional()?
.context("Media metadata file changed since indexing")
}
/// Publish probed tags only if no targeted media hint arrived since probe
/// admission. The check and tag update share one write transaction.
pub fn set_media_metadata_at_revision(
&mut self,
path: &str,
tags: &str,
duration_ms: u32,
size: u64,
mtime_ns: i64,
expected_revision: i64,
) -> Result<()> {
if tags.len() > 32768 {
bail!("Media tags exceed 32 KiB bound");
@@ -302,8 +344,8 @@ impl SearchIndex {
let tx = self.connection.transaction()?;
let id: Option<i64> = tx
.query_row(
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3 AND change_revision=?4",
params![path, i64::try_from(size)?, mtime_ns, expected_revision],
|r| r.get(0),
)
.optional()?;
@@ -411,6 +453,110 @@ impl SearchIndex {
if !root.is_dir() {
bail!("index root must be a directory");
}
self.reconcile_scopes(&root, std::slice::from_ref(&root), options, None, false)
}
/// Atomically reconcile at most 128 named files or directory subtrees.
/// Paths must already be absolute and component-normalized. Missing paths
/// remove only their matching root-owned records; an unavailable root fails
/// without touching the index. Symlink events/ancestors are rejected.
pub fn index_paths_with_options(
&mut self,
root: impl AsRef<Path>,
paths: &[PathBuf],
options: &IndexOptions,
) -> Result<IndexReport> {
self.index_paths_inner(root.as_ref(), paths, options, None)
}
/// Cancelled work rolls back the complete reconciliation transaction.
pub fn index_folder_cancellable(
&mut self,
root: impl AsRef<Path>,
options: &IndexOptions,
cancelled: &AtomicBool,
) -> Result<IndexReport> {
let root = fs::canonicalize(root).context("resolve index root")?;
if !root.is_dir() {
bail!("index root must be a directory");
}
self.reconcile_scopes(
&root,
std::slice::from_ref(&root),
options,
Some(cancelled),
false,
)
}
pub fn index_paths_cancellable(
&mut self,
root: impl AsRef<Path>,
paths: &[PathBuf],
options: &IndexOptions,
cancelled: &AtomicBool,
) -> Result<IndexReport> {
self.index_paths_inner(root.as_ref(), paths, options, Some(cancelled))
}
fn index_paths_inner(
&mut self,
root: &Path,
paths: &[PathBuf],
options: &IndexOptions,
cancelled: Option<&AtomicBool>,
) -> Result<IndexReport> {
check_index_cancelled(cancelled)?;
if paths.len() > 128 {
bail!("Targeted reconciliation exceeds 128-path batch limit");
}
let root = fs::canonicalize(root).context("resolve index root")?;
if !root.is_dir() {
bail!("index root must be a directory");
}
let mut scopes = Vec::new();
for path in paths {
validate_normalized_absolute(path)?;
let relative = path
.strip_prefix(&root)
.context("Targeted path lies outside index root")?;
let mut ancestor = root.clone();
for component in relative.components() {
ancestor.push(component.as_os_str());
match fs::symlink_metadata(&ancestor) {
Ok(metadata) if metadata.file_type().is_symlink() => {
bail!("Targeted path crosses a symlink")
}
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => break,
Err(error) => return Err(error).context("Inspect targeted path ancestor"),
}
}
scopes.push(path.clone());
}
scopes.sort();
scopes.dedup();
let mut reduced: Vec<PathBuf> = Vec::new();
for scope in scopes {
if !reduced.iter().any(|parent| scope.starts_with(parent)) {
reduced.push(scope);
}
}
self.reconcile_scopes(&root, &reduced, options, cancelled, true)
}
fn reconcile_scopes(
&mut self,
root: &Path,
scopes: &[PathBuf],
options: &IndexOptions,
cancelled: Option<&AtomicBool>,
media_hints: bool,
) -> Result<IndexReport> {
check_index_cancelled(cancelled)?;
for excluded in &options.excluded_paths {
validate_normalized_absolute(excluded)?;
}
let root_key = root.to_string_lossy().into_owned();
let transaction = self.connection.transaction()?;
transaction.execute_batch(
@@ -418,139 +564,205 @@ impl SearchIndex {
DELETE FROM seen_paths;",
)?;
let mut report = IndexReport::default();
for entry in WalkDir::new(&root)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
entry.depth() == 0
|| !entry.file_type().is_dir()
|| !options.excluded_directory_names.iter().any(|name| {
entry
.file_name()
.to_string_lossy()
.eq_ignore_ascii_case(name)
})
})
{
let entry = entry.context("directory traversal failed; reconciliation rolled back")?;
if !entry.file_type().is_file() {
for scope in scopes {
check_index_cancelled(cancelled)?;
match fs::symlink_metadata(scope) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error).context("Inspect reconciliation scope"),
Ok(metadata) if metadata.file_type().is_symlink() => {
bail!("Reconciliation scope became a symlink")
}
Ok(_) => {}
}
let relative = scope.strip_prefix(root)?;
let relative = if scope.is_file() {
relative.parent().unwrap_or(Path::new(""))
} else {
relative
};
let excluded = relative.components().any(|component| {
options.excluded_directory_names.iter().any(|name| {
component
.as_os_str()
.to_string_lossy()
.eq_ignore_ascii_case(name)
})
});
if excluded
|| options
.excluded_paths
.iter()
.any(|path| scope.starts_with(path))
{
continue;
}
if let Some(database) = &self.database_path {
let path_key = entry.path().to_string_lossy();
let db_key = database.to_string_lossy();
if path_key == db_key
|| path_key == format!("{db_key}-wal")
|| path_key == format!("{db_key}-shm")
{
continue;
}
}
let path = entry.path().to_string_lossy().into_owned();
let filename = entry.file_name().to_string_lossy().into_owned();
transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?;
report.discovered += 1;
let metadata = entry.metadata()?;
let size = i64::try_from(metadata.len()).context("file exceeds supported size")?;
let modified = metadata
.modified()?
.duration_since(UNIX_EPOCH)
.context("file timestamp predates Unix epoch")?;
let mtime = i64::try_from(modified.as_nanos())
.context("file timestamp exceeds supported range")?;
let extension = entry
.path()
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
for entry in WalkDir::new(scope)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
!options
.excluded_paths
.iter()
.any(|path| entry.path().starts_with(path))
&& (entry.depth() == 0
|| !entry.file_type().is_dir()
|| !options.excluded_directory_names.iter().any(|name| {
entry
.file_name()
.to_string_lossy()
.eq_ignore_ascii_case(name)
}))
})
{
report.unchanged += 1;
continue;
}
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
let reason: String = format!("{error:#}").chars().take(2048).collect();
(None, Some(reason))
check_index_cancelled(cancelled)?;
let entry =
entry.context("directory traversal failed; reconciliation rolled back")?;
if !entry.file_type().is_file() {
continue;
}
Err(error) => {
return Err(error)
.with_context(|| format!("read {path}; reconciliation rolled back"))
if let Some(database) = &self.database_path {
let path_key = entry.path().to_string_lossy();
let db_key = database.to_string_lossy();
if path_key == db_key
|| path_key == format!("{db_key}-wal")
|| path_key == format!("{db_key}-shm")
{
continue;
}
}
};
if extracted.is_none() {
report.skipped += 1;
}
let extracted = extracted.unwrap_or_default();
let content = extracted
.iter()
.map(|passage| passage.text.as_str())
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
let path = entry.path().to_string_lossy().into_owned();
let media_hint = media_hints && searchi_media::classify(entry.path()).is_some();
let filename = entry.file_name().to_string_lossy().into_owned();
transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?;
report.discovered += 1;
let metadata = entry.metadata()?;
let size = i64::try_from(metadata.len()).context("file exceeds supported size")?;
let modified = metadata
.modified()?
.duration_since(UNIX_EPOCH)
.context("file timestamp predates Unix epoch")?;
let mtime = i64::try_from(modified.as_nanos())
.context("file timestamp exceeds supported range")?;
let extension = entry
.path()
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
// A parser rejection may retain a filename-only record, but a
// filesystem read failure cannot certify this reconciliation.
if input
.as_ref()
.is_err_and(|error| error.chain().any(|cause| cause.is::<std::io::Error>()))
{
return Err(input.err().expect("Checked source read error"))
.with_context(|| format!("read {path}; reconciliation rolled back"));
}
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if !media_hint
&& previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
})
{
report.unchanged += 1;
continue;
}
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
let reason: String = format!("{error:#}").chars().take(2048).collect();
(None, Some(reason))
}
Err(error) => {
return Err(error)
.with_context(|| format!("read {path}; reconciliation rolled back"))
}
};
if extracted.is_none() {
report.skipped += 1;
}
let extracted = extracted.unwrap_or_default();
let content = extracted
.iter()
.map(|passage| passage.text.as_str())
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256,change_revision) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)
ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename,
content=excluded.content, size=excluded.size, mtime=excluded.mtime,
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash],
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256,
change_revision=files.change_revision+excluded.change_revision",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash, i64::from(media_hint)],
)?;
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
transaction.execute(
"INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,?3,?4)",
params![
passage.text,
path,
passage.source.kind,
passage.source.number
],
)?;
if media_hint {
transaction.execute("DELETE FROM media_metadata WHERE path=?1", [&path])?;
}
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
transaction.execute(
"INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,?3,?4)",
params![
passage.text,
path,
passage.source.kind,
passage.source.number
],
)?;
}
report.indexed += 1;
}
report.indexed += 1;
}
report.removed = transaction.execute(
"DELETE FROM files WHERE root=?1
AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)",
[&root_key],
)?;
// Recheck root availability before deleting records. A disconnected
// volume must never look like successful reconciliation of empty scopes.
let root_metadata =
fs::symlink_metadata(root).context("Recheck index root availability")?;
if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() {
bail!("Index root became unavailable");
}
for scope in scopes {
let key = scope.to_string_lossy();
let prefix = format!("{}{}", key, std::path::MAIN_SEPARATOR);
report.removed += transaction.execute(
"DELETE FROM files WHERE root=?1 AND (?4 OR path=?2 OR substr(path,1,length(?3))=?3)
AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)",
params![root_key, key, prefix, scope == root],
)?;
}
transaction.execute("DELETE FROM seen_paths", [])?;
check_index_cancelled(cancelled)?;
transaction.commit()?;
self.gram_frequencies.borrow_mut().clear();
Ok(report)
@@ -1164,6 +1376,28 @@ fn substring_anchor(characters: &[char], frequencies: &[(i64, String)]) -> Strin
characters[start..end].iter().collect()
}
fn check_index_cancelled(cancelled: Option<&AtomicBool>) -> Result<()> {
if cancelled.is_some_and(|flag| flag.load(Ordering::Acquire)) {
bail!("Index reconciliation cancelled; transaction rolled back");
}
Ok(())
}
fn validate_normalized_absolute(path: &Path) -> Result<()> {
if !path.is_absolute()
|| path.components().any(|c| {
matches!(
c,
std::path::Component::ParentDir | std::path::Component::CurDir
)
})
|| path.components().collect::<PathBuf>().as_os_str() != path.as_os_str()
{
bail!("Path must be absolute and component-normalized");
}
Ok(())
}
#[cfg(test)]
mod tests {
#[test]
crates/searchi-core/tests/index_options.rs
+1
View File
@@ -12,6 +12,7 @@ fn exclusions_database_and_unchanged_metadata() -> Result<()> {
fs::write(root.path().join("visible.txt"), "original content")?;
let mut index = SearchIndex::open(root.path().join("index.db"))?;
let options = IndexOptions {
excluded_paths: Vec::new(),
excluded_directory_names: vec![
".git".into(),
"node_modules".into(),
crates/searchi-core/tests/targeted_reconciliation.rs
+484
View File
@@ -0,0 +1,484 @@
//! Atomic bounded event reconciliation, independent of platform watcher delivery.
use anyhow::Result;
use searchi_core::{IndexOptions, SearchIndex};
use std::{fs, path::PathBuf, sync::atomic::AtomicBool};
#[test]
fn same_stat_overwrite_and_unrelated_files_are_preserved() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
let file = root.join("changed.txt");
fs::write(&file, "oldword alpha")?;
fs::write(root.join("other.txt"), "unrelated beta")?;
let root = fs::canonicalize(root)?;
let file = fs::canonicalize(file)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
let mtime = fs::metadata(&file)?.modified()?;
fs::write(&file, "newword alpha")?;
fs::File::options()
.write(true)
.open(&file)?
.set_times(fs::FileTimes::new().set_modified(mtime))?;
let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
assert_eq!(report.discovered, 1);
assert_eq!(report.indexed, 1);
assert_eq!(index.search("newword", 10)?.len(), 1);
assert!(index.search("oldword", 10)?.is_empty());
assert_eq!(index.search("unrelated", 10)?.len(), 1);
Ok(())
}
#[test]
fn rename_subtree_deletion_and_literal_boundaries_preserve_other_roots() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
let child = root.join("a_%");
fs::create_dir(&child)?;
let sibling = root.join("a_%suffix");
fs::create_dir(&sibling)?;
fs::write(child.join("old.txt"), "firstrecord")?;
fs::write(sibling.join("keep.txt"), "siblingrecord")?;
let other = temp.path().join("other");
fs::create_dir(&other)?;
fs::write(other.join("keep.txt"), "otherrootrecord")?;
let root = fs::canonicalize(root)?;
let child = root.join("a_%");
let old = child.join("old.txt");
let new = child.join("renamed.txt");
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
index.index_folder(&other)?;
fs::rename(&old, &new)?;
let report = index.index_paths_with_options(
&root,
&[old, new.clone(), new],
&IndexOptions::default(),
)?;
assert_eq!(report.indexed, 1);
assert_eq!(report.removed, 1);
assert_eq!(index.search("firstrecord", 10)?[0].filename, "renamed.txt");
fs::remove_dir_all(&child)?;
let report = index.index_paths_with_options(&root, &[child], &IndexOptions::default())?;
assert_eq!(report.removed, 1);
assert_eq!(index.search("siblingrecord", 10)?.len(), 1);
assert_eq!(index.search("otherrootrecord", 10)?.len(), 1);
Ok(())
}
#[test]
fn directory_events_reconcile_only_subtree_and_prune_exclusions() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::create_dir(root.join("nested"))?;
fs::create_dir(root.join("target"))?;
fs::write(root.join("nested/one.txt"), "oldnested")?;
fs::write(root.join("top.txt"), "toporiginal")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(root.join("index.db"))?;
index.index_folder(&root)?;
fs::remove_file(root.join("nested/one.txt"))?;
fs::write(root.join("nested/two.txt"), "newnested")?;
fs::write(root.join("top.txt"), "topchanged")?;
fs::write(root.join("target/excluded.txt"), "excludedcontent")?;
let report = index.index_paths_with_options(
&root,
&[
root.join("nested"),
root.join("nested/two.txt"),
root.join("target"),
],
&IndexOptions::default(),
)?;
assert_eq!(report.discovered, 1);
assert_eq!(report.removed, 1);
assert_eq!(index.search("newnested", 10)?.len(), 1);
assert_eq!(index.search("toporiginal", 10)?.len(), 1);
assert!(index.search("topchanged", 10)?.is_empty());
assert!(index.search("excludedcontent", 10)?.is_empty());
assert!(index.search("index.db", 10)?.is_empty());
Ok(())
}
#[test]
fn invalid_batches_and_failed_reads_roll_back_all_changes() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("a.txt"), "beforetransaction")?;
fs::write(root.join("z.txt"), "validlast")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
fs::write(root.join("a.txt"), "aftertransaction")?;
#[cfg(windows)]
let locked = {
use std::os::windows::fs::OpenOptionsExt;
fs::OpenOptions::new()
.read(true)
.share_mode(0)
.open(root.join("z.txt"))?
};
#[cfg(unix)]
let permissions = {
use std::os::unix::fs::PermissionsExt;
let previous = fs::metadata(root.join("z.txt"))?.permissions();
fs::set_permissions(root.join("z.txt"), fs::Permissions::from_mode(0))?;
previous
};
let failed = index.index_paths_with_options(
&root,
&[root.join("a.txt"), root.join("z.txt")],
&IndexOptions::default(),
);
#[cfg(windows)]
drop(locked);
#[cfg(unix)]
fs::set_permissions(root.join("z.txt"), permissions)?;
assert!(failed.is_err());
assert_eq!(index.search("beforetransaction", 10)?.len(), 1);
assert!(index.search("aftertransaction", 10)?.is_empty());
assert!(index
.index_paths_with_options(
&root,
&[temp.path().join("outside.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index
.index_paths_with_options(
&root,
&[PathBuf::from("relative.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index
.index_paths_with_options(
&root,
&[root.join("../outside.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index
.index_paths_with_options(
&root,
&vec![root.join("a.txt"); 129],
&IndexOptions::default()
)
.is_err());
Ok(())
}
#[test]
fn cancellation_and_unavailable_root_preserve_records() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("file.txt"), "preservedrecord")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
fs::remove_file(root.join("file.txt"))?;
let cancelled = AtomicBool::new(true);
assert!(index
.index_paths_cancellable(
&root,
&[root.join("file.txt")],
&IndexOptions::default(),
&cancelled
)
.is_err());
assert!(index
.index_folder_cancellable(&root, &IndexOptions::default(), &cancelled)
.is_err());
assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
fs::remove_dir(&root)?;
assert!(index
.index_paths_with_options(&root, &[root.join("file.txt")], &IndexOptions::default())
.is_err());
assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
Ok(())
}
#[cfg(unix)]
#[test]
fn symlink_ancestor_cannot_index_outside_content() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
let outside = temp.path().join("outside");
fs::create_dir(&outside)?;
fs::write(outside.join("secret.txt"), "secretoutside")?;
let root = fs::canonicalize(root)?;
std::os::unix::fs::symlink(&outside, root.join("link"))?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
assert!(index
.index_paths_with_options(
&root,
&[root.join("link/secret.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index.search("secretoutside", 10)?.is_empty());
Ok(())
}
#[test]
fn explicit_exclusions_remove_prior_records_without_prefix_leakage() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::create_dir(root.join("cache"))?;
fs::create_dir(root.join("cache-extra"))?;
fs::write(root.join("cache/hidden.txt"), "previouslyvisible")?;
fs::write(root.join("cache-extra/keep.txt"), "preservedneighbor")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
let mut options = IndexOptions::default();
options.excluded_paths.push(root.join("cache"));
let report = index.index_paths_with_options(&root, &[root.join("cache")], &options)?;
assert_eq!(report.removed, 1);
assert!(index.search("previouslyvisible", 10)?.is_empty());
assert_eq!(index.search("preservedneighbor", 10)?.len(), 1);
assert_eq!(
index.index_folder_with_options(&root, &options)?.discovered,
1
);
options.excluded_paths.push(PathBuf::from("relative"));
assert!(index.index_folder_with_options(&root, &options).is_err());
Ok(())
}
#[test]
fn independent_reader_remains_available_and_sees_atomic_update() -> Result<()> {
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
for number in 0..64 {
fs::write(root.join(format!("{number}.txt")), "oldversion")?;
}
let root = fs::canonicalize(root)?;
let db = temp.path().join("index.db");
let mut writer = SearchIndex::open(&db)?;
writer.index_folder(&root)?;
let reader = SearchIndex::open(&db)?;
for number in 0..64 {
fs::write(
root.join(format!("{number}.txt")),
format!("newversion {}", "content ".repeat(4000)),
)?;
}
let finished = Arc::new(AtomicBool::new(false));
let done = finished.clone();
std::thread::scope(|scope| -> Result<()> {
let task = scope.spawn(move || {
let result = writer.index_paths_with_options(
&root,
std::slice::from_ref(&root),
&IndexOptions::default(),
);
done.store(true, Ordering::Release);
result
});
let mut observations = 0;
while !finished.load(Ordering::Acquire) {
let count = reader.search("oldversion", 100)?.len();
assert!(
count == 64 || count == 0,
"Reader must never see partially committed records"
);
observations += 1;
}
task.join().unwrap()?;
assert!(observations > 0);
assert_eq!(reader.search("newversion", 100)?.len(), 64);
Ok(())
})?;
Ok(())
}
#[test]
fn cancellation_after_write_transaction_begins_rolls_back_batch() -> Result<()> {
use std::{
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
time::{Duration, Instant},
};
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
for number in 0..128 {
fs::write(root.join(format!("{number}.txt")), "oldcancelrecord")?;
}
let root = fs::canonicalize(root)?;
let db = temp.path().join("index.db");
let mut writer = SearchIndex::open(&db)?;
writer.index_folder(&root)?;
let reader = SearchIndex::open(&db)?;
for number in 0..128 {
fs::write(
root.join(format!("{number}.txt")),
format!("newcancelrecord {}", "new content ".repeat(8000)),
)?;
}
let probe = rusqlite::Connection::open(&db)?;
probe.busy_timeout(Duration::ZERO)?;
let cancelled = Arc::new(AtomicBool::new(false));
let flag = cancelled.clone();
std::thread::scope(|scope| -> Result<()> {
let indexing = scope
.spawn(move || writer.index_folder_cancellable(&root, &IndexOptions::default(), &flag));
let deadline = Instant::now() + Duration::from_secs(5);
let mut observed_write = false;
while Instant::now() < deadline {
match probe.execute_batch("BEGIN IMMEDIATE") {
Ok(()) => probe.execute_batch("ROLLBACK")?,
Err(rusqlite::Error::SqliteFailure(error, _))
if error.code == rusqlite::ErrorCode::DatabaseBusy =>
{
observed_write = true;
cancelled.store(true, Ordering::Release);
break;
}
Err(error) => return Err(error.into()),
}
std::thread::sleep(Duration::from_millis(1));
}
let result = indexing.join().unwrap();
assert!(
observed_write,
"Must observe the writer transaction before cancellation"
);
assert!(result.is_err());
assert_eq!(reader.search("oldcancelrecord", 200)?.len(), 128);
assert!(reader.search("newcancelrecord", 200)?.is_empty());
Ok(())
})?;
Ok(())
}
#[test]
fn same_stat_media_hint_advances_revision_and_removes_only_its_tags() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("changed.wav"), [1u8, 2, 3, 4])?;
fs::write(root.join("unrelated.wav"), [5u8, 6, 7, 8])?;
let root = fs::canonicalize(root)?;
let file = root.join("changed.wav");
let db = temp.path().join("index.db");
let mut index = SearchIndex::open(&db)?;
index.index_folder(&root)?;
let path = file.to_string_lossy().into_owned();
let (size, mtime) = index.file_signature(&path)?.unwrap();
index.set_media_metadata(&path, "stale artist metadata", 1000, size, mtime)?;
let original_mtime = fs::metadata(&file)?.modified()?;
fs::write(&file, [4u8, 3, 2, 1])?;
fs::OpenOptions::new()
.write(true)
.open(&file)?
.set_times(fs::FileTimes::new().set_modified(original_mtime))?;
assert_eq!(
index.index_folder(&root)?.unchanged,
2,
"Periodic metadata scan retains fast path"
);
let connection = rusqlite::Connection::open(&db)?;
let revisions = || -> Result<Vec<(String, i64)>> {
let mut statement =
connection.prepare("SELECT filename,change_revision FROM files ORDER BY filename")?;
let rows = statement.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
Ok(rows.collect::<rusqlite::Result<_>>()?)
};
assert_eq!(
revisions()?,
vec![("changed.wav".into(), 0), ("unrelated.wav".into(), 0)]
);
assert_eq!(index.search("stale artist", 10)?.len(), 1);
let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
assert_eq!(report.indexed, 1);
assert_eq!(report.unchanged, 0);
assert_eq!(
revisions()?,
vec![("changed.wav".into(), 1), ("unrelated.wav".into(), 0)]
);
assert!(index.search("stale artist", 10)?.is_empty());
assert_eq!(index.search("changed.wav", 10)?.len(), 1);
let report = index.index_paths_with_options(
&root,
std::slice::from_ref(&root),
&IndexOptions::default(),
)?;
assert_eq!(report.indexed, 2);
assert_eq!(
revisions()?,
vec![("changed.wav".into(), 2), ("unrelated.wav".into(), 1)]
);
Ok(())
}
#[test]
fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("document.txt"), "preserved migration text")?;
let db = temp.path().join("index.db");
let mut index = SearchIndex::open(&db)?;
index.index_folder(&root)?;
drop(index);
let connection = rusqlite::Connection::open(&db)?;
connection
.execute_batch("ALTER TABLE files DROP COLUMN change_revision; PRAGMA user_version=12;")?;
drop(connection);
let mut index = SearchIndex::open(&db)?;
assert_eq!(index.search("migration text", 10)?.len(), 1);
assert_eq!(index.index_folder(&root)?.unchanged, 1);
let connection = rusqlite::Connection::open(&db)?;
assert_eq!(
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
13
);
assert_eq!(
connection.query_row("SELECT change_revision FROM files", [], |row| row
.get::<_, i64>(0))?,
0
);
Ok(())
}
#[test]
fn metadata_probe_revision_rejects_tags_after_same_stat_hint() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("song.wav"), [1u8, 2, 3, 4])?;
let root = fs::canonicalize(root)?;
let file = root.join("song.wav");
let path = file.to_string_lossy().into_owned();
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
let (size, mtime) = index.file_signature(&path)?.unwrap();
let before = index.media_change_revision(&path, size, mtime)?;
index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
assert!(index
.set_media_metadata_at_revision(&path, "stale probe tags", 1000, size, mtime, before)
.is_err());
assert!(index.search("stale probe", 10)?.is_empty());
let current = index.media_change_revision(&path, size, mtime)?;
assert_eq!(current, before + 1);
index.set_media_metadata_at_revision(&path, "fresh probe tags", 1000, size, mtime, current)?;
assert_eq!(index.search("fresh probe", 10)?.len(), 1);
assert!(index.media_change_revision(&path, size + 1, mtime).is_err());
Ok(())
}