karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Watch selected folders and reconcile bounded changes with media freshness guards

Browse Source
This commit is contained in:
karl committed 2026-10-13 02:30:38 +02:00
1 parent 99748bd99f
commit 657de3de86
22 files changed
+2022 -149
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/searchi-core/Cargo.toml
+1
View File
@@ -5,6 +5,7 @@ edition.workspace = true
license.workspace = true
[dependencies]
searchi-media = { path = "../searchi-media" }
anyhow.workspace = true
serde.workspace = true
sha2 = "0.10"
crates/searchi-core/src/lib.rs
+359 -125
View File
@@ -17,11 +17,12 @@ use std::{
collections::{BTreeSet, HashMap},
fs,
path::{Path, PathBuf},
sync::atomic::{AtomicBool, Ordering},
time::UNIX_EPOCH,
};
use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 12;
const SCHEMA_VERSION: i64 = 13;
const MAX_RESULTS: usize = 1000;
#[derive(Debug, Default, Serialize)]
@@ -38,12 +39,15 @@ pub struct IndexReport {
#[derive(Debug, Clone)]
pub struct IndexOptions {
pub excluded_directory_names: Vec<String>,
/// Absolute component-normalized directory/file paths, matched by boundary.
pub excluded_paths: Vec<PathBuf>,
}
impl Default for IndexOptions {
fn default() -> Self {
Self {
excluded_directory_names: vec![".git".into(), "node_modules".into(), "target".into()],
excluded_paths: Vec::new(),
}
}
}
@@ -262,7 +266,17 @@ impl SearchIndex {
if !has_extraction_hash {
initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?;
}
initialization.execute_batch("PRAGMA user_version=12;")?;
let has_change_revision: bool = initialization.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='change_revision')",
[], |row| row.get(0),
)?;
if !has_change_revision {
initialization.execute(
"ALTER TABLE files ADD COLUMN change_revision INTEGER NOT NULL DEFAULT 0",
[],
)?;
}
initialization.execute_batch("PRAGMA user_version=13;")?;
initialization.commit()?;
}
Ok(Self {
@@ -295,6 +309,34 @@ impl SearchIndex {
duration_ms: u32,
size: u64,
mtime_ns: i64,
) -> Result<()> {
let revision = self.media_change_revision(path, size, mtime_ns)?;
self.set_media_metadata_at_revision(path, tags, duration_ms, size, mtime_ns, revision)
}
/// Capture before probing media. Event hints invalidate same-stat inputs
/// independently of their size and timestamp.
pub fn media_change_revision(&self, path: &str, size: u64, mtime_ns: i64) -> Result<i64> {
self.connection
.query_row(
"SELECT change_revision FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
|row| row.get(0),
)
.optional()?
.context("Media metadata file changed since indexing")
}
/// Publish probed tags only if no targeted media hint arrived since probe
/// admission. The check and tag update share one write transaction.
pub fn set_media_metadata_at_revision(
&mut self,
path: &str,
tags: &str,
duration_ms: u32,
size: u64,
mtime_ns: i64,
expected_revision: i64,
) -> Result<()> {
if tags.len() > 32768 {
bail!("Media tags exceed 32 KiB bound");
@@ -302,8 +344,8 @@ impl SearchIndex {
let tx = self.connection.transaction()?;
let id: Option<i64> = tx
.query_row(
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3 AND change_revision=?4",
params![path, i64::try_from(size)?, mtime_ns, expected_revision],
|r| r.get(0),
)
.optional()?;
@@ -411,6 +453,110 @@ impl SearchIndex {
if !root.is_dir() {
bail!("index root must be a directory");
}
self.reconcile_scopes(&root, std::slice::from_ref(&root), options, None, false)
}
/// Atomically reconcile at most 128 named files or directory subtrees.
/// Paths must already be absolute and component-normalized. Missing paths
/// remove only their matching root-owned records; an unavailable root fails
/// without touching the index. Symlink events/ancestors are rejected.
pub fn index_paths_with_options(
&mut self,
root: impl AsRef<Path>,
paths: &[PathBuf],
options: &IndexOptions,
) -> Result<IndexReport> {
self.index_paths_inner(root.as_ref(), paths, options, None)
}
/// Cancelled work rolls back the complete reconciliation transaction.
pub fn index_folder_cancellable(
&mut self,
root: impl AsRef<Path>,
options: &IndexOptions,
cancelled: &AtomicBool,
) -> Result<IndexReport> {
let root = fs::canonicalize(root).context("resolve index root")?;
if !root.is_dir() {
bail!("index root must be a directory");
}
self.reconcile_scopes(
&root,
std::slice::from_ref(&root),
options,
Some(cancelled),
false,
)
}
pub fn index_paths_cancellable(
&mut self,
root: impl AsRef<Path>,
paths: &[PathBuf],
options: &IndexOptions,
cancelled: &AtomicBool,
) -> Result<IndexReport> {
self.index_paths_inner(root.as_ref(), paths, options, Some(cancelled))
}
fn index_paths_inner(
&mut self,
root: &Path,
paths: &[PathBuf],
options: &IndexOptions,
cancelled: Option<&AtomicBool>,
) -> Result<IndexReport> {
check_index_cancelled(cancelled)?;
if paths.len() > 128 {
bail!("Targeted reconciliation exceeds 128-path batch limit");
}
let root = fs::canonicalize(root).context("resolve index root")?;
if !root.is_dir() {
bail!("index root must be a directory");
}
let mut scopes = Vec::new();
for path in paths {
validate_normalized_absolute(path)?;
let relative = path
.strip_prefix(&root)
.context("Targeted path lies outside index root")?;
let mut ancestor = root.clone();
for component in relative.components() {
ancestor.push(component.as_os_str());
match fs::symlink_metadata(&ancestor) {
Ok(metadata) if metadata.file_type().is_symlink() => {
bail!("Targeted path crosses a symlink")
}
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => break,
Err(error) => return Err(error).context("Inspect targeted path ancestor"),
}
}
scopes.push(path.clone());
}
scopes.sort();
scopes.dedup();
let mut reduced: Vec<PathBuf> = Vec::new();
for scope in scopes {
if !reduced.iter().any(|parent| scope.starts_with(parent)) {
reduced.push(scope);
}
}
self.reconcile_scopes(&root, &reduced, options, cancelled, true)
}
fn reconcile_scopes(
&mut self,
root: &Path,
scopes: &[PathBuf],
options: &IndexOptions,
cancelled: Option<&AtomicBool>,
media_hints: bool,
) -> Result<IndexReport> {
check_index_cancelled(cancelled)?;
for excluded in &options.excluded_paths {
validate_normalized_absolute(excluded)?;
}
let root_key = root.to_string_lossy().into_owned();
let transaction = self.connection.transaction()?;
transaction.execute_batch(
@@ -418,139 +564,205 @@ impl SearchIndex {
DELETE FROM seen_paths;",
)?;
let mut report = IndexReport::default();
for entry in WalkDir::new(&root)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
entry.depth() == 0
|| !entry.file_type().is_dir()
|| !options.excluded_directory_names.iter().any(|name| {
entry
.file_name()
.to_string_lossy()
.eq_ignore_ascii_case(name)
})
})
{
let entry = entry.context("directory traversal failed; reconciliation rolled back")?;
if !entry.file_type().is_file() {
for scope in scopes {
check_index_cancelled(cancelled)?;
match fs::symlink_metadata(scope) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error).context("Inspect reconciliation scope"),
Ok(metadata) if metadata.file_type().is_symlink() => {
bail!("Reconciliation scope became a symlink")
}
Ok(_) => {}
}
let relative = scope.strip_prefix(root)?;
let relative = if scope.is_file() {
relative.parent().unwrap_or(Path::new(""))
} else {
relative
};
let excluded = relative.components().any(|component| {
options.excluded_directory_names.iter().any(|name| {
component
.as_os_str()
.to_string_lossy()
.eq_ignore_ascii_case(name)
})
});
if excluded
|| options
.excluded_paths
.iter()
.any(|path| scope.starts_with(path))
{
continue;
}
if let Some(database) = &self.database_path {
let path_key = entry.path().to_string_lossy();
let db_key = database.to_string_lossy();
if path_key == db_key
|| path_key == format!("{db_key}-wal")
|| path_key == format!("{db_key}-shm")
{
continue;
}
}
let path = entry.path().to_string_lossy().into_owned();
let filename = entry.file_name().to_string_lossy().into_owned();
transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?;
report.discovered += 1;
let metadata = entry.metadata()?;
let size = i64::try_from(metadata.len()).context("file exceeds supported size")?;
let modified = metadata
.modified()?
.duration_since(UNIX_EPOCH)
.context("file timestamp predates Unix epoch")?;
let mtime = i64::try_from(modified.as_nanos())
.context("file timestamp exceeds supported range")?;
let extension = entry
.path()
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
for entry in WalkDir::new(scope)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
!options
.excluded_paths
.iter()
.any(|path| entry.path().starts_with(path))
&& (entry.depth() == 0
|| !entry.file_type().is_dir()
|| !options.excluded_directory_names.iter().any(|name| {
entry
.file_name()
.to_string_lossy()
.eq_ignore_ascii_case(name)
}))
})
{
report.unchanged += 1;
continue;
}
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
let reason: String = format!("{error:#}").chars().take(2048).collect();
(None, Some(reason))
check_index_cancelled(cancelled)?;
let entry =
entry.context("directory traversal failed; reconciliation rolled back")?;
if !entry.file_type().is_file() {
continue;
}
Err(error) => {
return Err(error)
.with_context(|| format!("read {path}; reconciliation rolled back"))
if let Some(database) = &self.database_path {
let path_key = entry.path().to_string_lossy();
let db_key = database.to_string_lossy();
if path_key == db_key
|| path_key == format!("{db_key}-wal")
|| path_key == format!("{db_key}-shm")
{
continue;
}
}
};
if extracted.is_none() {
report.skipped += 1;
}
let extracted = extracted.unwrap_or_default();
let content = extracted
.iter()
.map(|passage| passage.text.as_str())
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
let path = entry.path().to_string_lossy().into_owned();
let media_hint = media_hints && searchi_media::classify(entry.path()).is_some();
let filename = entry.file_name().to_string_lossy().into_owned();
transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?;
report.discovered += 1;
let metadata = entry.metadata()?;
let size = i64::try_from(metadata.len()).context("file exceeds supported size")?;
let modified = metadata
.modified()?
.duration_since(UNIX_EPOCH)
.context("file timestamp predates Unix epoch")?;
let mtime = i64::try_from(modified.as_nanos())
.context("file timestamp exceeds supported range")?;
let extension = entry
.path()
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
// A parser rejection may retain a filename-only record, but a
// filesystem read failure cannot certify this reconciliation.
if input
.as_ref()
.is_err_and(|error| error.chain().any(|cause| cause.is::<std::io::Error>()))
{
return Err(input.err().expect("Checked source read error"))
.with_context(|| format!("read {path}; reconciliation rolled back"));
}
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if !media_hint
&& previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
})
{
report.unchanged += 1;
continue;
}
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
let reason: String = format!("{error:#}").chars().take(2048).collect();
(None, Some(reason))
}
Err(error) => {
return Err(error)
.with_context(|| format!("read {path}; reconciliation rolled back"))
}
};
if extracted.is_none() {
report.skipped += 1;
}
let extracted = extracted.unwrap_or_default();
let content = extracted
.iter()
.map(|passage| passage.text.as_str())
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256,change_revision) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)
ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename,
content=excluded.content, size=excluded.size, mtime=excluded.mtime,
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash],
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256,
change_revision=files.change_revision+excluded.change_revision",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash, i64::from(media_hint)],
)?;
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
transaction.execute(
"INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,?3,?4)",
params![
passage.text,
path,
passage.source.kind,
passage.source.number
],
)?;
if media_hint {
transaction.execute("DELETE FROM media_metadata WHERE path=?1", [&path])?;
}
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
transaction.execute(
"INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,?3,?4)",
params![
passage.text,
path,
passage.source.kind,
passage.source.number
],
)?;
}
report.indexed += 1;
}
report.indexed += 1;
}
report.removed = transaction.execute(
"DELETE FROM files WHERE root=?1
AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)",
[&root_key],
)?;
// Recheck root availability before deleting records. A disconnected
// volume must never look like successful reconciliation of empty scopes.
let root_metadata =
fs::symlink_metadata(root).context("Recheck index root availability")?;
if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() {
bail!("Index root became unavailable");
}
for scope in scopes {
let key = scope.to_string_lossy();
let prefix = format!("{}{}", key, std::path::MAIN_SEPARATOR);
report.removed += transaction.execute(
"DELETE FROM files WHERE root=?1 AND (?4 OR path=?2 OR substr(path,1,length(?3))=?3)
AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)",
params![root_key, key, prefix, scope == root],
)?;
}
transaction.execute("DELETE FROM seen_paths", [])?;
check_index_cancelled(cancelled)?;
transaction.commit()?;
self.gram_frequencies.borrow_mut().clear();
Ok(report)
@@ -1164,6 +1376,28 @@ fn substring_anchor(characters: &[char], frequencies: &[(i64, String)]) -> Strin
characters[start..end].iter().collect()
}
fn check_index_cancelled(cancelled: Option<&AtomicBool>) -> Result<()> {
if cancelled.is_some_and(|flag| flag.load(Ordering::Acquire)) {
bail!("Index reconciliation cancelled; transaction rolled back");
}
Ok(())
}
fn validate_normalized_absolute(path: &Path) -> Result<()> {
if !path.is_absolute()
|| path.components().any(|c| {
matches!(
c,
std::path::Component::ParentDir | std::path::Component::CurDir
)
})
|| path.components().collect::<PathBuf>().as_os_str() != path.as_os_str()
{
bail!("Path must be absolute and component-normalized");
}
Ok(())
}
#[cfg(test)]
mod tests {
#[test]
crates/searchi-core/tests/index_options.rs
+1
View File
@@ -12,6 +12,7 @@ fn exclusions_database_and_unchanged_metadata() -> Result<()> {
fs::write(root.path().join("visible.txt"), "original content")?;
let mut index = SearchIndex::open(root.path().join("index.db"))?;
let options = IndexOptions {
excluded_paths: Vec::new(),
excluded_directory_names: vec![
".git".into(),
"node_modules".into(),
crates/searchi-core/tests/targeted_reconciliation.rs
+484
View File
@@ -0,0 +1,484 @@
//! Atomic bounded event reconciliation, independent of platform watcher delivery.
use anyhow::Result;
use searchi_core::{IndexOptions, SearchIndex};
use std::{fs, path::PathBuf, sync::atomic::AtomicBool};
#[test]
fn same_stat_overwrite_and_unrelated_files_are_preserved() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
let file = root.join("changed.txt");
fs::write(&file, "oldword alpha")?;
fs::write(root.join("other.txt"), "unrelated beta")?;
let root = fs::canonicalize(root)?;
let file = fs::canonicalize(file)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
let mtime = fs::metadata(&file)?.modified()?;
fs::write(&file, "newword alpha")?;
fs::File::options()
.write(true)
.open(&file)?
.set_times(fs::FileTimes::new().set_modified(mtime))?;
let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
assert_eq!(report.discovered, 1);
assert_eq!(report.indexed, 1);
assert_eq!(index.search("newword", 10)?.len(), 1);
assert!(index.search("oldword", 10)?.is_empty());
assert_eq!(index.search("unrelated", 10)?.len(), 1);
Ok(())
}
#[test]
fn rename_subtree_deletion_and_literal_boundaries_preserve_other_roots() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
let child = root.join("a_%");
fs::create_dir(&child)?;
let sibling = root.join("a_%suffix");
fs::create_dir(&sibling)?;
fs::write(child.join("old.txt"), "firstrecord")?;
fs::write(sibling.join("keep.txt"), "siblingrecord")?;
let other = temp.path().join("other");
fs::create_dir(&other)?;
fs::write(other.join("keep.txt"), "otherrootrecord")?;
let root = fs::canonicalize(root)?;
let child = root.join("a_%");
let old = child.join("old.txt");
let new = child.join("renamed.txt");
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
index.index_folder(&other)?;
fs::rename(&old, &new)?;
let report = index.index_paths_with_options(
&root,
&[old, new.clone(), new],
&IndexOptions::default(),
)?;
assert_eq!(report.indexed, 1);
assert_eq!(report.removed, 1);
assert_eq!(index.search("firstrecord", 10)?[0].filename, "renamed.txt");
fs::remove_dir_all(&child)?;
let report = index.index_paths_with_options(&root, &[child], &IndexOptions::default())?;
assert_eq!(report.removed, 1);
assert_eq!(index.search("siblingrecord", 10)?.len(), 1);
assert_eq!(index.search("otherrootrecord", 10)?.len(), 1);
Ok(())
}
#[test]
fn directory_events_reconcile_only_subtree_and_prune_exclusions() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::create_dir(root.join("nested"))?;
fs::create_dir(root.join("target"))?;
fs::write(root.join("nested/one.txt"), "oldnested")?;
fs::write(root.join("top.txt"), "toporiginal")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(root.join("index.db"))?;
index.index_folder(&root)?;
fs::remove_file(root.join("nested/one.txt"))?;
fs::write(root.join("nested/two.txt"), "newnested")?;
fs::write(root.join("top.txt"), "topchanged")?;
fs::write(root.join("target/excluded.txt"), "excludedcontent")?;
let report = index.index_paths_with_options(
&root,
&[
root.join("nested"),
root.join("nested/two.txt"),
root.join("target"),
],
&IndexOptions::default(),
)?;
assert_eq!(report.discovered, 1);
assert_eq!(report.removed, 1);
assert_eq!(index.search("newnested", 10)?.len(), 1);
assert_eq!(index.search("toporiginal", 10)?.len(), 1);
assert!(index.search("topchanged", 10)?.is_empty());
assert!(index.search("excludedcontent", 10)?.is_empty());
assert!(index.search("index.db", 10)?.is_empty());
Ok(())
}
#[test]
fn invalid_batches_and_failed_reads_roll_back_all_changes() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("a.txt"), "beforetransaction")?;
fs::write(root.join("z.txt"), "validlast")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
fs::write(root.join("a.txt"), "aftertransaction")?;
#[cfg(windows)]
let locked = {
use std::os::windows::fs::OpenOptionsExt;
fs::OpenOptions::new()
.read(true)
.share_mode(0)
.open(root.join("z.txt"))?
};
#[cfg(unix)]
let permissions = {
use std::os::unix::fs::PermissionsExt;
let previous = fs::metadata(root.join("z.txt"))?.permissions();
fs::set_permissions(root.join("z.txt"), fs::Permissions::from_mode(0))?;
previous
};
let failed = index.index_paths_with_options(
&root,
&[root.join("a.txt"), root.join("z.txt")],
&IndexOptions::default(),
);
#[cfg(windows)]
drop(locked);
#[cfg(unix)]
fs::set_permissions(root.join("z.txt"), permissions)?;
assert!(failed.is_err());
assert_eq!(index.search("beforetransaction", 10)?.len(), 1);
assert!(index.search("aftertransaction", 10)?.is_empty());
assert!(index
.index_paths_with_options(
&root,
&[temp.path().join("outside.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index
.index_paths_with_options(
&root,
&[PathBuf::from("relative.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index
.index_paths_with_options(
&root,
&[root.join("../outside.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index
.index_paths_with_options(
&root,
&vec![root.join("a.txt"); 129],
&IndexOptions::default()
)
.is_err());
Ok(())
}
#[test]
fn cancellation_and_unavailable_root_preserve_records() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("file.txt"), "preservedrecord")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
fs::remove_file(root.join("file.txt"))?;
let cancelled = AtomicBool::new(true);
assert!(index
.index_paths_cancellable(
&root,
&[root.join("file.txt")],
&IndexOptions::default(),
&cancelled
)
.is_err());
assert!(index
.index_folder_cancellable(&root, &IndexOptions::default(), &cancelled)
.is_err());
assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
fs::remove_dir(&root)?;
assert!(index
.index_paths_with_options(&root, &[root.join("file.txt")], &IndexOptions::default())
.is_err());
assert_eq!(index.search("preservedrecord", 10)?.len(), 1);
Ok(())
}
#[cfg(unix)]
#[test]
fn symlink_ancestor_cannot_index_outside_content() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
let outside = temp.path().join("outside");
fs::create_dir(&outside)?;
fs::write(outside.join("secret.txt"), "secretoutside")?;
let root = fs::canonicalize(root)?;
std::os::unix::fs::symlink(&outside, root.join("link"))?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
assert!(index
.index_paths_with_options(
&root,
&[root.join("link/secret.txt")],
&IndexOptions::default()
)
.is_err());
assert!(index.search("secretoutside", 10)?.is_empty());
Ok(())
}
#[test]
fn explicit_exclusions_remove_prior_records_without_prefix_leakage() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::create_dir(root.join("cache"))?;
fs::create_dir(root.join("cache-extra"))?;
fs::write(root.join("cache/hidden.txt"), "previouslyvisible")?;
fs::write(root.join("cache-extra/keep.txt"), "preservedneighbor")?;
let root = fs::canonicalize(root)?;
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
let mut options = IndexOptions::default();
options.excluded_paths.push(root.join("cache"));
let report = index.index_paths_with_options(&root, &[root.join("cache")], &options)?;
assert_eq!(report.removed, 1);
assert!(index.search("previouslyvisible", 10)?.is_empty());
assert_eq!(index.search("preservedneighbor", 10)?.len(), 1);
assert_eq!(
index.index_folder_with_options(&root, &options)?.discovered,
1
);
options.excluded_paths.push(PathBuf::from("relative"));
assert!(index.index_folder_with_options(&root, &options).is_err());
Ok(())
}
#[test]
fn independent_reader_remains_available_and_sees_atomic_update() -> Result<()> {
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
for number in 0..64 {
fs::write(root.join(format!("{number}.txt")), "oldversion")?;
}
let root = fs::canonicalize(root)?;
let db = temp.path().join("index.db");
let mut writer = SearchIndex::open(&db)?;
writer.index_folder(&root)?;
let reader = SearchIndex::open(&db)?;
for number in 0..64 {
fs::write(
root.join(format!("{number}.txt")),
format!("newversion {}", "content ".repeat(4000)),
)?;
}
let finished = Arc::new(AtomicBool::new(false));
let done = finished.clone();
std::thread::scope(|scope| -> Result<()> {
let task = scope.spawn(move || {
let result = writer.index_paths_with_options(
&root,
std::slice::from_ref(&root),
&IndexOptions::default(),
);
done.store(true, Ordering::Release);
result
});
let mut observations = 0;
while !finished.load(Ordering::Acquire) {
let count = reader.search("oldversion", 100)?.len();
assert!(
count == 64 || count == 0,
"Reader must never see partially committed records"
);
observations += 1;
}
task.join().unwrap()?;
assert!(observations > 0);
assert_eq!(reader.search("newversion", 100)?.len(), 64);
Ok(())
})?;
Ok(())
}
#[test]
fn cancellation_after_write_transaction_begins_rolls_back_batch() -> Result<()> {
use std::{
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
time::{Duration, Instant},
};
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
for number in 0..128 {
fs::write(root.join(format!("{number}.txt")), "oldcancelrecord")?;
}
let root = fs::canonicalize(root)?;
let db = temp.path().join("index.db");
let mut writer = SearchIndex::open(&db)?;
writer.index_folder(&root)?;
let reader = SearchIndex::open(&db)?;
for number in 0..128 {
fs::write(
root.join(format!("{number}.txt")),
format!("newcancelrecord {}", "new content ".repeat(8000)),
)?;
}
let probe = rusqlite::Connection::open(&db)?;
probe.busy_timeout(Duration::ZERO)?;
let cancelled = Arc::new(AtomicBool::new(false));
let flag = cancelled.clone();
std::thread::scope(|scope| -> Result<()> {
let indexing = scope
.spawn(move || writer.index_folder_cancellable(&root, &IndexOptions::default(), &flag));
let deadline = Instant::now() + Duration::from_secs(5);
let mut observed_write = false;
while Instant::now() < deadline {
match probe.execute_batch("BEGIN IMMEDIATE") {
Ok(()) => probe.execute_batch("ROLLBACK")?,
Err(rusqlite::Error::SqliteFailure(error, _))
if error.code == rusqlite::ErrorCode::DatabaseBusy =>
{
observed_write = true;
cancelled.store(true, Ordering::Release);
break;
}
Err(error) => return Err(error.into()),
}
std::thread::sleep(Duration::from_millis(1));
}
let result = indexing.join().unwrap();
assert!(
observed_write,
"Must observe the writer transaction before cancellation"
);
assert!(result.is_err());
assert_eq!(reader.search("oldcancelrecord", 200)?.len(), 128);
assert!(reader.search("newcancelrecord", 200)?.is_empty());
Ok(())
})?;
Ok(())
}
#[test]
fn same_stat_media_hint_advances_revision_and_removes_only_its_tags() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("changed.wav"), [1u8, 2, 3, 4])?;
fs::write(root.join("unrelated.wav"), [5u8, 6, 7, 8])?;
let root = fs::canonicalize(root)?;
let file = root.join("changed.wav");
let db = temp.path().join("index.db");
let mut index = SearchIndex::open(&db)?;
index.index_folder(&root)?;
let path = file.to_string_lossy().into_owned();
let (size, mtime) = index.file_signature(&path)?.unwrap();
index.set_media_metadata(&path, "stale artist metadata", 1000, size, mtime)?;
let original_mtime = fs::metadata(&file)?.modified()?;
fs::write(&file, [4u8, 3, 2, 1])?;
fs::OpenOptions::new()
.write(true)
.open(&file)?
.set_times(fs::FileTimes::new().set_modified(original_mtime))?;
assert_eq!(
index.index_folder(&root)?.unchanged,
2,
"Periodic metadata scan retains fast path"
);
let connection = rusqlite::Connection::open(&db)?;
let revisions = || -> Result<Vec<(String, i64)>> {
let mut statement =
connection.prepare("SELECT filename,change_revision FROM files ORDER BY filename")?;
let rows = statement.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
Ok(rows.collect::<rusqlite::Result<_>>()?)
};
assert_eq!(
revisions()?,
vec![("changed.wav".into(), 0), ("unrelated.wav".into(), 0)]
);
assert_eq!(index.search("stale artist", 10)?.len(), 1);
let report = index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
assert_eq!(report.indexed, 1);
assert_eq!(report.unchanged, 0);
assert_eq!(
revisions()?,
vec![("changed.wav".into(), 1), ("unrelated.wav".into(), 0)]
);
assert!(index.search("stale artist", 10)?.is_empty());
assert_eq!(index.search("changed.wav", 10)?.len(), 1);
let report = index.index_paths_with_options(
&root,
std::slice::from_ref(&root),
&IndexOptions::default(),
)?;
assert_eq!(report.indexed, 2);
assert_eq!(
revisions()?,
vec![("changed.wav".into(), 2), ("unrelated.wav".into(), 1)]
);
Ok(())
}
#[test]
fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("document.txt"), "preserved migration text")?;
let db = temp.path().join("index.db");
let mut index = SearchIndex::open(&db)?;
index.index_folder(&root)?;
drop(index);
let connection = rusqlite::Connection::open(&db)?;
connection
.execute_batch("ALTER TABLE files DROP COLUMN change_revision; PRAGMA user_version=12;")?;
drop(connection);
let mut index = SearchIndex::open(&db)?;
assert_eq!(index.search("migration text", 10)?.len(), 1);
assert_eq!(index.index_folder(&root)?.unchanged, 1);
let connection = rusqlite::Connection::open(&db)?;
assert_eq!(
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
13
);
assert_eq!(
connection.query_row("SELECT change_revision FROM files", [], |row| row
.get::<_, i64>(0))?,
0
);
Ok(())
}
#[test]
fn metadata_probe_revision_rejects_tags_after_same_stat_hint() -> Result<()> {
let temp = tempfile::tempdir()?;
let root = temp.path().join("root");
fs::create_dir(&root)?;
fs::write(root.join("song.wav"), [1u8, 2, 3, 4])?;
let root = fs::canonicalize(root)?;
let file = root.join("song.wav");
let path = file.to_string_lossy().into_owned();
let mut index = SearchIndex::open(temp.path().join("index.db"))?;
index.index_folder(&root)?;
let (size, mtime) = index.file_signature(&path)?.unwrap();
let before = index.media_change_revision(&path, size, mtime)?;
index.index_paths_with_options(&root, &[file], &IndexOptions::default())?;
assert!(index
.set_media_metadata_at_revision(&path, "stale probe tags", 1000, size, mtime, before)
.is_err());
assert!(index.search("stale probe", 10)?.is_empty());
let current = index.media_change_revision(&path, size, mtime)?;
assert_eq!(current, before + 1);
index.set_media_metadata_at_revision(&path, "fresh probe tags", 1000, size, mtime, current)?;
assert_eq!(index.search("fresh probe", 10)?.len(), 1);
assert!(index.media_change_revision(&path, size + 1, mtime).is_err());
Ok(())
}
crates/searchi-media/src/lib.rs
+56 -2
View File
@@ -93,6 +93,29 @@ mod tests {
Ok(())
}
#[test]
fn owned_frame_scratch_validates_parent_and_cleans_batch() -> Result<()> {
let root = tempfile::tempdir()?;
let parent = root.path().join("owned scratch");
let limits = MediaLimits {
scratch_directory: Some(parent.clone()),
..Default::default()
};
let batch = frame_scratch(&limits)?;
assert_eq!(batch.path().parent(), Some(parent.as_path()));
let batch_path = batch.path().to_owned();
fs::write(batch.path().join("frame.png"), b"owned temporary bytes")?;
drop(batch);
assert!(!batch_path.exists());
assert!(parent.is_dir());
let relative = MediaLimits {
scratch_directory: Some(PathBuf::from("relative")),
..Default::default()
};
assert!(frame_scratch(&relative).is_err());
Ok(())
}
#[test]
fn identity_rejects_changes_and_oversized_input() -> Result<()> {
let temp = tempfile::tempdir()?;
@@ -170,10 +193,19 @@ mod tests {
])
.arg(&video);
run(command, Duration::from_secs(30), 4096, 65536)?;
let limits = MediaLimits::default();
let scratch = temp.path().join("owned scratch");
let limits = MediaLimits {
scratch_directory: Some(scratch.clone()),
..Default::default()
};
let probe = probe(&tools, &video, &limits)?;
assert!(probe.has_video && probe.has_audio);
let frames = decode_frames(&tools, &video, &probe, 2000, false, &limits)?;
assert!(frames.directory().starts_with(&scratch));
assert!(frames
.frames
.iter()
.all(|frame| frame.path.starts_with(&scratch)));
assert_eq!(frames.frames.len(), 1);
assert!((1900..=2100).contains(&frames.frames[0].timestamp_ms));
let audio = decode_audio(
@@ -312,6 +344,10 @@ pub struct MediaLimits {
pub audio_overlap_ms: u32,
pub max_audio_windows: usize,
pub command_timeout: Duration,
/// Owned decoded-frame scratch parent. Desktop callers place this below
/// excluded application data to avoid indexing their own temporary frames.
/// None preserves the normal operating-system temporary directory.
pub scratch_directory: Option<PathBuf>,
}
impl Default for MediaLimits {
fn default() -> Self {
@@ -325,6 +361,7 @@ impl Default for MediaLimits {
audio_overlap_ms: 2000,
max_audio_windows: 600,
command_timeout: Duration::from_secs(45),
scratch_directory: None,
}
}
}
@@ -615,6 +652,23 @@ impl FrameBatch {
self.directory.path()
}
}
fn frame_scratch(limits: &MediaLimits) -> Result<tempfile::TempDir> {
match &limits.scratch_directory {
Some(parent) => {
ensure!(
parent.is_absolute(),
"Media scratch directory must be absolute"
);
fs::create_dir_all(parent).context("Create owned media scratch directory")?;
tempfile::Builder::new()
.prefix("searchi-frames-")
.tempdir_in(parent)
.context("Create owned frame batch")
}
None => tempfile::tempdir().context("Create frame batch"),
}
}
pub fn decode_frames(
tools: &FfmpegTools,
path: &Path,
@@ -623,7 +677,7 @@ pub fn decode_frames(
include_scene: bool,
limits: &MediaLimits,
) -> Result<FrameBatch> {
let directory = tempfile::tempdir()?;
let directory = frame_scratch(limits)?;
let mut command = Command::new(&tools.ffmpeg);
command.args(["-protocol_whitelist", "file,pipe"]);
command.args(["-nostdin","-hide_banner","-loglevel","info","-threads","1","-hwaccel","none","-copyts","-ss"])
crates/searchi-service/src/lib.rs
+1 -1
View File
@@ -272,7 +272,7 @@ impl HybridSearch {
});
}
ensure!(
(6..=7).contains(&version),
(6..=8).contains(&version),
"Unsupported semantic schema version {version}"
);
let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
crates/searchi-service/src/media.rs
+12 -3
View File
@@ -141,6 +141,11 @@ impl HybridSearch {
task: &SourceTask,
work: &mut BuildWork,
) -> Result<()> {
let change_revision = self.lexical.media_change_revision(
&path.to_string_lossy(),
identity.bytes,
identity.modified_ns,
)?;
if let Some(count) = self
.store
.reuse_media_source(generation, task, id, identity)?
@@ -152,7 +157,7 @@ impl HybridSearch {
if kind != MediaKind::Image {
// Metadata remains part of music search after path remapping.
let metadata = probe(tools, path, limits)?;
self.refresh_media_metadata(path, identity, &metadata)?;
self.refresh_media_metadata(path, identity, &metadata, change_revision)?;
}
identity.ensure_unchanged()?;
work.report.chunks += count;
@@ -169,6 +174,7 @@ impl HybridSearch {
tools,
limits,
identity,
change_revision,
&mut work.report,
)?;
self.store
@@ -182,6 +188,7 @@ impl HybridSearch {
path: &Path,
identity: &MediaIdentity,
metadata: &MediaProbe,
change_revision: i64,
) -> Result<()> {
let tags = metadata
.tags
@@ -189,12 +196,13 @@ impl HybridSearch {
.map(|(k, v)| format!("{k}: {v}"))
.collect::<Vec<_>>()
.join("\n");
self.lexical.set_media_metadata(
self.lexical.set_media_metadata_at_revision(
&path.to_string_lossy(),
&tags,
metadata.duration_ms,
identity.bytes,
identity.modified_ns,
change_revision,
)
}
@@ -208,6 +216,7 @@ impl HybridSearch {
tools: &FfmpegTools,
limits: &MediaLimits,
identity: &MediaIdentity,
change_revision: i64,
report: &mut EmbeddingReport,
) -> Result<()> {
let mut ordinal = 0;
@@ -228,7 +237,7 @@ impl HybridSearch {
report.chunks += 1;
} else {
let metadata = probe(tools, path, limits)?;
self.refresh_media_metadata(path, identity, &metadata)?;
self.refresh_media_metadata(path, identity, &metadata, change_revision)?;
if metadata.has_video && kind == MediaKind::Video {
let mut seen = std::collections::HashSet::new();
let mut scenes = 0;
crates/searchi-service/src/store.rs
+3 -3
View File
@@ -111,7 +111,7 @@ impl Store {
|row| row.get(0),
)?;
ensure!(
(1..=7).contains(&version),
(1..=8).contains(&version),
"Unsupported semantic schema version {version}"
);
initialization.execute_batch("
@@ -273,7 +273,7 @@ impl Store {
)
};
let bytes: Vec<u8> = vector.iter().flat_map(|v| v.to_le_bytes()).collect();
self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12)",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,file_change_revision) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12,(SELECT change_revision FROM files WHERE path=?4))",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
Ok(())
}
@@ -991,7 +991,7 @@ fn collect_eligibility(
};
let mut statement=connection.prepare(&format!("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime))
JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime AND c.file_change_revision=f.change_revision))
AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
crates/searchi-service/src/store/incremental.rs
+10 -5
View File
@@ -22,6 +22,11 @@ pub(super) fn migrate(connection: &Connection) -> Result<()> {
"INTEGER NOT NULL DEFAULT 0",
),
("semantic_chunks", "task_key", "TEXT"),
(
"semantic_chunks",
"file_change_revision",
"INTEGER NOT NULL DEFAULT 0",
),
] {
let exists: bool = connection.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)",
@@ -43,7 +48,7 @@ pub(super) fn migrate(connection: &Connection) -> Result<()> {
PRIMARY KEY(generation,task_key));
CREATE INDEX IF NOT EXISTS semantic_task_reuse ON semantic_tasks(source_hash,title_hash,policy,source_type,state);
CREATE INDEX IF NOT EXISTS semantic_chunk_task ON semantic_chunks(generation,task_key);
UPDATE semantic_meta SET schema_version=7 WHERE singleton=1;")?;
UPDATE semantic_meta SET schema_version=8 WHERE singleton=1;")?;
Ok(())
}
@@ -294,8 +299,8 @@ impl Store {
for (key, donor) in mappings {
tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts,expected_chunks,message)
SELECT ?1,d.task_key,d.path,d.source_type,d.source_hash,d.title_hash,d.policy,d.passage_id,'done',old.attempts,old.expected_chunks,NULL FROM semantic_discovery_tasks d JOIN semantic_prune_tasks old ON old.task_key=?3 WHERE d.task_key=?2",params![generation,key,donor])?;
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision)
SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key,COALESCE((SELECT change_revision FROM files WHERE path=d.path),c.file_change_revision) FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
}
tx.execute(
"DELETE FROM semantic_media_errors WHERE generation=?1",
@@ -530,8 +535,8 @@ impl Store {
"DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2",
params![generation, task.key],
)?;
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8 FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision)
SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8,COALESCE((SELECT change_revision FROM files WHERE path=?3),file_change_revision) FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count as i64,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?;
ensure!(changed == 1, "Missing source task during reuse");
if task.source_type == "media" {
crates/searchi-service/tests/incremental_google_cpu.rs
+48 -2
View File
@@ -321,14 +321,39 @@ fn genuine_google_mixed_media_reuse_checks_bytes_despite_identical_file_metadata
.open(&image)?
.set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?;
assert_eq!(fs::metadata(&image)?.modified()?, metadata.modified()?);
// Core intentionally sees unchanged size/mtime: semantic reuse must still
// hash actual media bytes before accepting a completion manifest.
// Full metadata reconciliation intentionally retains its media fast path.
// A native event must durably invalidate old media even with identical stats.
lexical.index_folder(&root)?;
let canonical_root = fs::canonicalize(&root)?;
let image_hint = fs::canonicalize(&image)?;
lexical.index_paths_with_options(
&canonical_root,
std::slice::from_ref(&image_hint),
&Default::default(),
)?;
assert!(HybridSearch::embedding_status(&db)?.stale);
let options = SearchOptions {
extension: Some("bmp".into()),
..Default::default()
};
assert!(
service.search("a red square", 10, &options)?.is_empty(),
"The hinted old image must be hidden until actual byte-verified reindexing"
);
service.index_embeddings_with_media(&tools, &limits)?;
let changed = calls.lock().unwrap().clone();
assert_eq!(changed.documents, initial.documents);
assert_eq!(changed.audio, initial.audio);
assert_eq!(changed.images, initial.images + 1);
assert!(!HybridSearch::embedding_status(&db)?.stale);
let response = service.search("a green square", 10, &options)?;
assert_eq!(response[0].filename, "color.bmp");
let connection = Connection::open(&db)?;
let revisions: (i64,i64) = connection.query_row(
"SELECT f.change_revision,c.file_change_revision FROM files f JOIN semantic_chunks c ON c.path=f.path WHERE f.path=?1 AND c.generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1)",
[image_hint.to_string_lossy().as_ref()], |row| Ok((row.get(0)?,row.get(1)?)),
)?;
assert_eq!(revisions, (1, 1));
let latest = chunks(&db)?;
for chunk in first.iter().filter(|chunk| !chunk.0.ends_with("color.bmp")) {
assert!(
@@ -348,6 +373,27 @@ fn genuine_google_mixed_media_reuse_checks_bytes_despite_identical_file_metadata
.unwrap()
.6
);
// Unchanged bytes still need fresh eligibility after a hint, but the SHA-
// verified completed source should reuse vectors and remap revision tokens.
let audio_hint = fs::canonicalize(root.join("tone.wav"))?;
lexical.index_paths_with_options(
&canonical_root,
&[image_hint.clone(), audio_hint.clone()],
&Default::default(),
)?;
assert!(service.search("a green square", 10, &options)?.is_empty());
let reused = service.index_embeddings_with_media(&tools, &limits)?;
assert_eq!(reused.reused_sources, 5);
assert_eq!(*calls.lock().unwrap(), changed);
assert_eq!(
service.search("a green square", 10, &options)?[0].filename,
"color.bmp"
);
let audio_revisions: (i64,i64) = connection.query_row(
"SELECT f.change_revision,c.file_change_revision FROM files f JOIN semantic_chunks c ON c.path=f.path WHERE f.path=?1 AND c.generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1)",
[audio_hint.to_string_lossy().as_ref()], |row| Ok((row.get(0)?,row.get(1)?)),
)?;
assert_eq!(audio_revisions, (1, 1));
Ok(())
}