karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Watch selected folders and reconcile bounded changes with media freshness guards

Browse Source
This commit is contained in:
karl committed 2026-10-13 02:30:38 +02:00
1 parent 99748bd99f
commit 657de3de86
22 files changed
+2022 -149
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/searchi-core/src/lib.rs
+359 -125
View File
@@ -17,11 +17,12 @@ use std::{
collections::{BTreeSet, HashMap},
fs,
path::{Path, PathBuf},
sync::atomic::{AtomicBool, Ordering},
time::UNIX_EPOCH,
};
use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 12;
const SCHEMA_VERSION: i64 = 13;
const MAX_RESULTS: usize = 1000;
#[derive(Debug, Default, Serialize)]
@@ -38,12 +39,15 @@ pub struct IndexReport {
#[derive(Debug, Clone)]
pub struct IndexOptions {
pub excluded_directory_names: Vec<String>,
/// Absolute component-normalized directory/file paths, matched by boundary.
pub excluded_paths: Vec<PathBuf>,
}
impl Default for IndexOptions {
fn default() -> Self {
Self {
excluded_directory_names: vec![".git".into(), "node_modules".into(), "target".into()],
excluded_paths: Vec::new(),
}
}
}
@@ -262,7 +266,17 @@ impl SearchIndex {
if !has_extraction_hash {
initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?;
}
initialization.execute_batch("PRAGMA user_version=12;")?;
let has_change_revision: bool = initialization.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='change_revision')",
[], |row| row.get(0),
)?;
if !has_change_revision {
initialization.execute(
"ALTER TABLE files ADD COLUMN change_revision INTEGER NOT NULL DEFAULT 0",
[],
)?;
}
initialization.execute_batch("PRAGMA user_version=13;")?;
initialization.commit()?;
}
Ok(Self {
@@ -295,6 +309,34 @@ impl SearchIndex {
duration_ms: u32,
size: u64,
mtime_ns: i64,
) -> Result<()> {
let revision = self.media_change_revision(path, size, mtime_ns)?;
self.set_media_metadata_at_revision(path, tags, duration_ms, size, mtime_ns, revision)
}
/// Capture before probing media. Event hints invalidate same-stat inputs
/// independently of their size and timestamp.
pub fn media_change_revision(&self, path: &str, size: u64, mtime_ns: i64) -> Result<i64> {
self.connection
.query_row(
"SELECT change_revision FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
|row| row.get(0),
)
.optional()?
.context("Media metadata file changed since indexing")
}
/// Publish probed tags only if no targeted media hint arrived since probe
/// admission. The check and tag update share one write transaction.
pub fn set_media_metadata_at_revision(
&mut self,
path: &str,
tags: &str,
duration_ms: u32,
size: u64,
mtime_ns: i64,
expected_revision: i64,
) -> Result<()> {
if tags.len() > 32768 {
bail!("Media tags exceed 32 KiB bound");
@@ -302,8 +344,8 @@ impl SearchIndex {
let tx = self.connection.transaction()?;
let id: Option<i64> = tx
.query_row(
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3 AND change_revision=?4",
params![path, i64::try_from(size)?, mtime_ns, expected_revision],
|r| r.get(0),
)
.optional()?;
@@ -411,6 +453,110 @@ impl SearchIndex {
if !root.is_dir() {
bail!("index root must be a directory");
}
self.reconcile_scopes(&root, std::slice::from_ref(&root), options, None, false)
}
/// Atomically reconcile at most 128 named files or directory subtrees.
/// Paths must already be absolute and component-normalized. Missing paths
/// remove only their matching root-owned records; an unavailable root fails
/// without touching the index. Symlink events/ancestors are rejected.
pub fn index_paths_with_options(
&mut self,
root: impl AsRef<Path>,
paths: &[PathBuf],
options: &IndexOptions,
) -> Result<IndexReport> {
self.index_paths_inner(root.as_ref(), paths, options, None)
}
/// Cancelled work rolls back the complete reconciliation transaction.
pub fn index_folder_cancellable(
&mut self,
root: impl AsRef<Path>,
options: &IndexOptions,
cancelled: &AtomicBool,
) -> Result<IndexReport> {
let root = fs::canonicalize(root).context("resolve index root")?;
if !root.is_dir() {
bail!("index root must be a directory");
}
self.reconcile_scopes(
&root,
std::slice::from_ref(&root),
options,
Some(cancelled),
false,
)
}
pub fn index_paths_cancellable(
&mut self,
root: impl AsRef<Path>,
paths: &[PathBuf],
options: &IndexOptions,
cancelled: &AtomicBool,
) -> Result<IndexReport> {
self.index_paths_inner(root.as_ref(), paths, options, Some(cancelled))
}
fn index_paths_inner(
&mut self,
root: &Path,
paths: &[PathBuf],
options: &IndexOptions,
cancelled: Option<&AtomicBool>,
) -> Result<IndexReport> {
check_index_cancelled(cancelled)?;
if paths.len() > 128 {
bail!("Targeted reconciliation exceeds 128-path batch limit");
}
let root = fs::canonicalize(root).context("resolve index root")?;
if !root.is_dir() {
bail!("index root must be a directory");
}
let mut scopes = Vec::new();
for path in paths {
validate_normalized_absolute(path)?;
let relative = path
.strip_prefix(&root)
.context("Targeted path lies outside index root")?;
let mut ancestor = root.clone();
for component in relative.components() {
ancestor.push(component.as_os_str());
match fs::symlink_metadata(&ancestor) {
Ok(metadata) if metadata.file_type().is_symlink() => {
bail!("Targeted path crosses a symlink")
}
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => break,
Err(error) => return Err(error).context("Inspect targeted path ancestor"),
}
}
scopes.push(path.clone());
}
scopes.sort();
scopes.dedup();
let mut reduced: Vec<PathBuf> = Vec::new();
for scope in scopes {
if !reduced.iter().any(|parent| scope.starts_with(parent)) {
reduced.push(scope);
}
}
self.reconcile_scopes(&root, &reduced, options, cancelled, true)
}
fn reconcile_scopes(
&mut self,
root: &Path,
scopes: &[PathBuf],
options: &IndexOptions,
cancelled: Option<&AtomicBool>,
media_hints: bool,
) -> Result<IndexReport> {
check_index_cancelled(cancelled)?;
for excluded in &options.excluded_paths {
validate_normalized_absolute(excluded)?;
}
let root_key = root.to_string_lossy().into_owned();
let transaction = self.connection.transaction()?;
transaction.execute_batch(
@@ -418,139 +564,205 @@ impl SearchIndex {
DELETE FROM seen_paths;",
)?;
let mut report = IndexReport::default();
for entry in WalkDir::new(&root)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
entry.depth() == 0
|| !entry.file_type().is_dir()
|| !options.excluded_directory_names.iter().any(|name| {
entry
.file_name()
.to_string_lossy()
.eq_ignore_ascii_case(name)
})
})
{
let entry = entry.context("directory traversal failed; reconciliation rolled back")?;
if !entry.file_type().is_file() {
for scope in scopes {
check_index_cancelled(cancelled)?;
match fs::symlink_metadata(scope) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error).context("Inspect reconciliation scope"),
Ok(metadata) if metadata.file_type().is_symlink() => {
bail!("Reconciliation scope became a symlink")
}
Ok(_) => {}
}
let relative = scope.strip_prefix(root)?;
let relative = if scope.is_file() {
relative.parent().unwrap_or(Path::new(""))
} else {
relative
};
let excluded = relative.components().any(|component| {
options.excluded_directory_names.iter().any(|name| {
component
.as_os_str()
.to_string_lossy()
.eq_ignore_ascii_case(name)
})
});
if excluded
|| options
.excluded_paths
.iter()
.any(|path| scope.starts_with(path))
{
continue;
}
if let Some(database) = &self.database_path {
let path_key = entry.path().to_string_lossy();
let db_key = database.to_string_lossy();
if path_key == db_key
|| path_key == format!("{db_key}-wal")
|| path_key == format!("{db_key}-shm")
{
continue;
}
}
let path = entry.path().to_string_lossy().into_owned();
let filename = entry.file_name().to_string_lossy().into_owned();
transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?;
report.discovered += 1;
let metadata = entry.metadata()?;
let size = i64::try_from(metadata.len()).context("file exceeds supported size")?;
let modified = metadata
.modified()?
.duration_since(UNIX_EPOCH)
.context("file timestamp predates Unix epoch")?;
let mtime = i64::try_from(modified.as_nanos())
.context("file timestamp exceeds supported range")?;
let extension = entry
.path()
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
for entry in WalkDir::new(scope)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
!options
.excluded_paths
.iter()
.any(|path| entry.path().starts_with(path))
&& (entry.depth() == 0
|| !entry.file_type().is_dir()
|| !options.excluded_directory_names.iter().any(|name| {
entry
.file_name()
.to_string_lossy()
.eq_ignore_ascii_case(name)
}))
})
{
report.unchanged += 1;
continue;
}
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
let reason: String = format!("{error:#}").chars().take(2048).collect();
(None, Some(reason))
check_index_cancelled(cancelled)?;
let entry =
entry.context("directory traversal failed; reconciliation rolled back")?;
if !entry.file_type().is_file() {
continue;
}
Err(error) => {
return Err(error)
.with_context(|| format!("read {path}; reconciliation rolled back"))
if let Some(database) = &self.database_path {
let path_key = entry.path().to_string_lossy();
let db_key = database.to_string_lossy();
if path_key == db_key
|| path_key == format!("{db_key}-wal")
|| path_key == format!("{db_key}-shm")
{
continue;
}
}
};
if extracted.is_none() {
report.skipped += 1;
}
let extracted = extracted.unwrap_or_default();
let content = extracted
.iter()
.map(|passage| passage.text.as_str())
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
let path = entry.path().to_string_lossy().into_owned();
let media_hint = media_hints && searchi_media::classify(entry.path()).is_some();
let filename = entry.file_name().to_string_lossy().into_owned();
transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?;
report.discovered += 1;
let metadata = entry.metadata()?;
let size = i64::try_from(metadata.len()).context("file exceeds supported size")?;
let modified = metadata
.modified()?
.duration_since(UNIX_EPOCH)
.context("file timestamp predates Unix epoch")?;
let mtime = i64::try_from(modified.as_nanos())
.context("file timestamp exceeds supported range")?;
let extension = entry
.path()
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
// A parser rejection may retain a filename-only record, but a
// filesystem read failure cannot certify this reconciliation.
if input
.as_ref()
.is_err_and(|error| error.chain().any(|cause| cause.is::<std::io::Error>()))
{
return Err(input.err().expect("Checked source read error"))
.with_context(|| format!("read {path}; reconciliation rolled back"));
}
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if !media_hint
&& previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
})
{
report.unchanged += 1;
continue;
}
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
let reason: String = format!("{error:#}").chars().take(2048).collect();
(None, Some(reason))
}
Err(error) => {
return Err(error)
.with_context(|| format!("read {path}; reconciliation rolled back"))
}
};
if extracted.is_none() {
report.skipped += 1;
}
let extracted = extracted.unwrap_or_default();
let content = extracted
.iter()
.map(|passage| passage.text.as_str())
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256,change_revision) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)
ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename,
content=excluded.content, size=excluded.size, mtime=excluded.mtime,
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash],
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256,
change_revision=files.change_revision+excluded.change_revision",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash, i64::from(media_hint)],
)?;
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
transaction.execute(
"INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,?3,?4)",
params![
passage.text,
path,
passage.source.kind,
passage.source.number
],
)?;
if media_hint {
transaction.execute("DELETE FROM media_metadata WHERE path=?1", [&path])?;
}
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
transaction.execute(
"INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,?3,?4)",
params![
passage.text,
path,
passage.source.kind,
passage.source.number
],
)?;
}
report.indexed += 1;
}
report.indexed += 1;
}
report.removed = transaction.execute(
"DELETE FROM files WHERE root=?1
AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)",
[&root_key],
)?;
// Recheck root availability before deleting records. A disconnected
// volume must never look like successful reconciliation of empty scopes.
let root_metadata =
fs::symlink_metadata(root).context("Recheck index root availability")?;
if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() {
bail!("Index root became unavailable");
}
for scope in scopes {
let key = scope.to_string_lossy();
let prefix = format!("{}{}", key, std::path::MAIN_SEPARATOR);
report.removed += transaction.execute(
"DELETE FROM files WHERE root=?1 AND (?4 OR path=?2 OR substr(path,1,length(?3))=?3)
AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)",
params![root_key, key, prefix, scope == root],
)?;
}
transaction.execute("DELETE FROM seen_paths", [])?;
check_index_cancelled(cancelled)?;
transaction.commit()?;
self.gram_frequencies.borrow_mut().clear();
Ok(report)
@@ -1164,6 +1376,28 @@ fn substring_anchor(characters: &[char], frequencies: &[(i64, String)]) -> Strin
characters[start..end].iter().collect()
}
fn check_index_cancelled(cancelled: Option<&AtomicBool>) -> Result<()> {
if cancelled.is_some_and(|flag| flag.load(Ordering::Acquire)) {
bail!("Index reconciliation cancelled; transaction rolled back");
}
Ok(())
}
fn validate_normalized_absolute(path: &Path) -> Result<()> {
if !path.is_absolute()
|| path.components().any(|c| {
matches!(
c,
std::path::Component::ParentDir | std::path::Component::CurDir
)
})
|| path.components().collect::<PathBuf>().as_os_str() != path.as_os_str()
{
bail!("Path must be absolute and component-normalized");
}
Ok(())
}
#[cfg(test)]
mod tests {
#[test]