Watch selected folders and reconcile bounded changes with media freshness guards
This commit is contained in:
1 parent
99748bd99f
commit
657de3de86
22 files changed
+2022
-149
No files matched your search
Generated
+79
@@ -1345,6 +1345,15 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fsevent-sys"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.34"
|
||||
@@ -2133,6 +2142,26 @@ dependencies = [
|
||||
"cfb",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inotify"
|
||||
version = "0.11.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"inotify-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inotify-sys"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.2.2"
|
||||
@@ -2355,6 +2384,26 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kqueue"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea"
|
||||
dependencies = [
|
||||
"kqueue-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kqueue-sys"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libappindicator"
|
||||
version = "0.9.0"
|
||||
@@ -2562,6 +2611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -2648,6 +2698,33 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "notify"
|
||||
version = "8.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"fsevent-sys",
|
||||
"inotify",
|
||||
"kqueue",
|
||||
"libc",
|
||||
"log",
|
||||
"mio",
|
||||
"notify-types",
|
||||
"walkdir",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "notify-types"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
@@ -3718,6 +3795,7 @@ dependencies = [
|
||||
"lopdf",
|
||||
"quick-xml",
|
||||
"rusqlite",
|
||||
"searchi-media",
|
||||
"serde",
|
||||
"sha2 0.10.9",
|
||||
"walkdir",
|
||||
@@ -3730,6 +3808,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"image",
|
||||
"notify",
|
||||
"searchi-ai",
|
||||
"searchi-assets",
|
||||
"searchi-core",
|
||||
|
||||
@@ -15,6 +15,7 @@ tauri-build = { version = "2", features = [] }
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
notify = "=8.2.0"
|
||||
tauri = { version = "2", features = ["protocol-asset"] }
|
||||
tauri-plugin-dialog = "2"
|
||||
tauri-plugin-opener = "2"
|
||||
|
||||
@@ -1,6 +1,44 @@
|
||||
//! Explicit data-directory isolation for portable use and native release QA.
|
||||
//! Ordinary launches continue to use the operating system's app-data directory.
|
||||
use std::{ffi::OsString, io, path::PathBuf};
|
||||
use std::{
|
||||
ffi::OsString,
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// Match canonical watched paths, including Windows verbatim prefixes, even
|
||||
/// before a first-use model or scratch directory exists.
|
||||
pub fn canonical_exclusion(path: &Path) -> io::Result<PathBuf> {
|
||||
if !path.is_absolute() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"Excluded directory must be absolute",
|
||||
));
|
||||
}
|
||||
let mut ancestor = path.to_path_buf();
|
||||
let mut suffix = Vec::new();
|
||||
loop {
|
||||
match std::fs::canonicalize(&ancestor) {
|
||||
Ok(mut resolved) => {
|
||||
for part in suffix.iter().rev() {
|
||||
resolved.push(part);
|
||||
}
|
||||
return Ok(resolved);
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||
let name = ancestor.file_name().ok_or(error)?;
|
||||
suffix.push(name.to_os_string());
|
||||
if !ancestor.pop() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
"Excluded directory has no existing ancestor",
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve(default: PathBuf, override_path: Option<OsString>) -> io::Result<PathBuf> {
|
||||
let Some(value) = override_path else {
|
||||
@@ -20,6 +58,23 @@ pub fn resolve(default: PathBuf, override_path: Option<OsString>) -> io::Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn missing_exclusion_matches_canonical_root_without_creating_directories() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let missing = root.path().join("Models").join("MediaScratch");
|
||||
let canonical = std::fs::canonicalize(root.path()).unwrap();
|
||||
let exclusion = canonical_exclusion(&missing).unwrap();
|
||||
assert_eq!(exclusion, canonical.join("Models/MediaScratch"));
|
||||
assert!(exclusion.starts_with(&canonical));
|
||||
assert!(!missing.exists());
|
||||
assert_eq!(canonical_exclusion(root.path()).unwrap(), canonical);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_exclusion_is_rejected() {
|
||||
assert!(canonical_exclusion(Path::new("relative/cache")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_directory_isolates_settings_and_index_without_changing_default() {
|
||||
let default = std::env::temp_dir().join("searchi-default");
|
||||
|
||||
@@ -8,6 +8,7 @@ mod data_directory;
|
||||
mod model_download;
|
||||
mod semantic_commands;
|
||||
mod settings;
|
||||
mod watching;
|
||||
|
||||
use ai_commands::{generate_ai, prepare_ai_source, AiSession};
|
||||
use searchi_core::metrics::{MetricsMode, SearchMetrics, SearchResponse, TraceCollector};
|
||||
@@ -38,6 +39,8 @@ struct AppState {
|
||||
settings: Arc<Mutex<SettingsStore>>,
|
||||
shortcut_status: String,
|
||||
semantic: Arc<SemanticState>,
|
||||
watching: Arc<watching::WatchService>,
|
||||
index_options: searchi_core::IndexOptions,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -85,6 +88,7 @@ struct DesktopStatus {
|
||||
settings: Settings,
|
||||
indexing: bool,
|
||||
shortcut_status: String,
|
||||
watching: Vec<watching::WatchStatus>,
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
@@ -98,6 +102,7 @@ fn desktop_status(state: State<'_, AppState>) -> Result<DesktopStatus, String> {
|
||||
.clone(),
|
||||
indexing: state.indexing.load(Ordering::Acquire),
|
||||
shortcut_status: state.shortcut_status.clone(),
|
||||
watching: state.watching.status(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -362,15 +367,18 @@ async fn index_folder(root: String, state: State<'_, AppState>) -> Result<IndexR
|
||||
let database = state.database.clone();
|
||||
let settings = state.settings.clone();
|
||||
let lexical = state.lexical.clone();
|
||||
let watching = state.watching.clone();
|
||||
let options = state.index_options.clone();
|
||||
tauri::async_runtime::spawn_blocking(move || {
|
||||
let _guard = guard;
|
||||
settings
|
||||
.lock()
|
||||
.map_err(|_| "Settings are unavailable.")?
|
||||
.add_root(std::path::Path::new(&root))?;
|
||||
watching.add_root(std::fs::canonicalize(&root).map_err(|error| error.to_string())?)?;
|
||||
let mut index = SearchIndex::open(database).map_err(|error| error.to_string())?;
|
||||
let report = index
|
||||
.index_folder(root)
|
||||
.index_folder_with_options(root, &options)
|
||||
.map_err(|error| error.to_string())?;
|
||||
index.prewarm().map_err(|error| error.to_string())?;
|
||||
*lexical
|
||||
@@ -382,6 +390,66 @@ async fn index_folder(root: String, state: State<'_, AppState>) -> Result<IndexR
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
|
||||
/// Each reconciliation owns its connection; retained readers stay usable while
|
||||
/// extraction runs. The watcher retains dirty work when the shared gate is busy.
|
||||
fn start_folder_watching(
|
||||
app: tauri::AppHandle,
|
||||
database: PathBuf,
|
||||
lexical: Arc<Mutex<SearchIndex>>,
|
||||
indexing: Arc<AtomicBool>,
|
||||
roots: Vec<PathBuf>,
|
||||
options: searchi_core::IndexOptions,
|
||||
) -> Result<watching::WatchService, String> {
|
||||
watching::WatchService::start(roots, options.excluded_paths.clone(), move |request| {
|
||||
if indexing
|
||||
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_err()
|
||||
{
|
||||
return watching::ScanOutcome::Busy;
|
||||
}
|
||||
let guard = IndexGuard(indexing.clone());
|
||||
let _ = app.emit("folder-indexing-changed", ());
|
||||
let result = (|| -> Result<bool, String> {
|
||||
let mut index = SearchIndex::open(&database).map_err(|error| error.to_string())?;
|
||||
let report = match request.paths {
|
||||
Some(paths) => index.index_paths_cancellable(
|
||||
&request.root,
|
||||
&paths,
|
||||
&options,
|
||||
&request.cancelled,
|
||||
),
|
||||
None if request.media_hint => index.index_paths_cancellable(
|
||||
&request.root,
|
||||
std::slice::from_ref(&request.root),
|
||||
&options,
|
||||
&request.cancelled,
|
||||
),
|
||||
None => index.index_folder_cancellable(&request.root, &options, &request.cancelled),
|
||||
}
|
||||
.map_err(|error| error.to_string())?;
|
||||
let changed = report.indexed > 0 || report.removed > 0;
|
||||
if changed && !request.cancelled.load(Ordering::Acquire) {
|
||||
index.prewarm().map_err(|error| error.to_string())?;
|
||||
*lexical
|
||||
.lock()
|
||||
.map_err(|_| "Keyword engine is unavailable.")? = index;
|
||||
}
|
||||
Ok(changed)
|
||||
})();
|
||||
drop(guard);
|
||||
let _ = app.emit("folder-indexing-changed", ());
|
||||
match result {
|
||||
Ok(changed) => {
|
||||
if changed {
|
||||
let _ = app.emit("search-index-changed", ());
|
||||
}
|
||||
watching::ScanOutcome::Completed
|
||||
}
|
||||
Err(error) => watching::ScanOutcome::Failed(error),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn main() {
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_dialog::init())
|
||||
@@ -455,7 +523,25 @@ fn main() {
|
||||
"The media decoder is missing from this installation. Repair or reinstall searchi.".into()
|
||||
})
|
||||
})();
|
||||
let semantic = Arc::new(SemanticState::new(model_directory, worker, media_tools));
|
||||
let mut ignored = vec![directory.clone(), model_directory.clone()];
|
||||
if let Some(parent) = worker.parent().filter(|path| path.is_absolute()) {
|
||||
ignored.push(parent.to_path_buf());
|
||||
}
|
||||
let index_options = searchi_core::IndexOptions {
|
||||
excluded_paths: ignored.iter()
|
||||
.map(|path| data_directory::canonical_exclusion(path))
|
||||
.collect::<std::io::Result<Vec<_>>>()?,
|
||||
..Default::default()
|
||||
};
|
||||
let indexing = Arc::new(AtomicBool::new(false));
|
||||
let watching = Arc::new(start_folder_watching(
|
||||
app.handle().clone(), database.clone(), lexical.clone(), indexing.clone(),
|
||||
settings.value.roots.clone(), index_options.clone(),
|
||||
).map_err(std::io::Error::other)?);
|
||||
let semantic = Arc::new(
|
||||
SemanticState::new(model_directory, worker, media_tools)
|
||||
.with_media_scratch(directory.join("MediaScratch")),
|
||||
);
|
||||
semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
|
||||
#[cfg(target_os = "macos")]
|
||||
let shortcut = "Shift+Super+Space";
|
||||
@@ -475,10 +561,12 @@ fn main() {
|
||||
ai: Arc::new(AiSession::default()),
|
||||
database,
|
||||
lexical,
|
||||
indexing: Arc::new(AtomicBool::new(false)),
|
||||
indexing,
|
||||
settings: Arc::new(Mutex::new(settings)),
|
||||
shortcut_status,
|
||||
semantic,
|
||||
watching,
|
||||
index_options,
|
||||
});
|
||||
Ok(())
|
||||
})
|
||||
@@ -500,8 +588,15 @@ fn main() {
|
||||
hybrid_search,
|
||||
hybrid_search_measured
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
.expect("Searchi desktop could not start");
|
||||
.build(tauri::generate_context!())
|
||||
.expect("Searchi desktop could not start")
|
||||
.run(|app, event| {
|
||||
if matches!(event, tauri::RunEvent::Exit) {
|
||||
// Release the worker-held AppHandle and owned watcher before
|
||||
// application state is torn down.
|
||||
app.state::<AppState>().watching.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -29,6 +29,7 @@ pub struct SemanticState {
|
||||
inference_budget: InferenceBudget,
|
||||
lifecycle: Mutex<()>,
|
||||
media_tools: Option<searchi_media::FfmpegTools>,
|
||||
media_limits: searchi_media::MediaLimits,
|
||||
media_setup_error: Option<String>,
|
||||
}
|
||||
|
||||
@@ -58,10 +59,17 @@ impl SemanticState {
|
||||
inference_budget: InferenceBudget::new(2),
|
||||
lifecycle: Mutex::new(()),
|
||||
media_tools,
|
||||
media_limits: searchi_media::MediaLimits::default(),
|
||||
media_setup_error,
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep generated video frames below the application's excluded data tree.
|
||||
pub fn with_media_scratch(mut self, directory: PathBuf) -> Self {
|
||||
self.media_limits.scratch_directory = Some(directory);
|
||||
self
|
||||
}
|
||||
|
||||
fn ensure_idle(&self) -> Result<(), String> {
|
||||
if self.embedding.load(Ordering::Acquire)
|
||||
|| self.downloading.load(Ordering::Acquire)
|
||||
@@ -335,7 +343,7 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
|
||||
let tools = semantic.media_tools.as_ref().ok_or_else(|| {
|
||||
anyhow::anyhow!("The media decoder is missing from this installation. Repair or reinstall searchi.")
|
||||
})?;
|
||||
service.index_embeddings_with_media(tools, &searchi_media::MediaLimits::default())
|
||||
service.index_embeddings_with_media(tools, &semantic.media_limits)
|
||||
})()
|
||||
.map_err(|error| error.to_string());
|
||||
// The publisher shares its prepared immutable ANN before activation.
|
||||
|
||||
@@ -0,0 +1,644 @@
|
||||
//! Native notifications are bounded hints; periodic reconciliation is authoritative.
|
||||
use notify::{Event, EventKind, RecursiveMode, Watcher};
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc, Condvar, Mutex,
|
||||
},
|
||||
thread::{self, JoinHandle},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
const MAX_ROOTS: usize = 64;
|
||||
const MAX_PATHS: usize = 128;
|
||||
|
||||
pub struct ScanRequest {
|
||||
pub root: PathBuf,
|
||||
pub paths: Option<Vec<PathBuf>>,
|
||||
pub cancelled: Arc<AtomicBool>,
|
||||
pub media_hint: bool,
|
||||
}
|
||||
pub enum ScanOutcome {
|
||||
Completed,
|
||||
Busy,
|
||||
Failed(String),
|
||||
}
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct WatchStatus {
|
||||
pub root: PathBuf,
|
||||
pub attached: bool,
|
||||
pub pending: bool,
|
||||
pub error: Option<String>,
|
||||
pub watch_error: Option<String>,
|
||||
}
|
||||
#[derive(Clone, Copy)]
|
||||
struct Timing {
|
||||
debounce: Duration,
|
||||
max_delay: Duration,
|
||||
periodic: Duration,
|
||||
retry: Duration,
|
||||
}
|
||||
impl Default for Timing {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
debounce: Duration::from_millis(750),
|
||||
max_delay: Duration::from_secs(5),
|
||||
periodic: Duration::from_secs(300),
|
||||
retry: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
struct Root {
|
||||
paths: BTreeSet<PathBuf>,
|
||||
full: bool,
|
||||
media_hint: bool,
|
||||
ever_attached: bool,
|
||||
first: Option<Instant>,
|
||||
last: Instant,
|
||||
next_scan: Instant,
|
||||
retry_at: Instant,
|
||||
attach_retry: Instant,
|
||||
failures: u32,
|
||||
attached: bool,
|
||||
error: Option<String>,
|
||||
watch_error: Option<String>,
|
||||
}
|
||||
impl Root {
|
||||
fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
paths: BTreeSet::new(),
|
||||
full: true,
|
||||
media_hint: false,
|
||||
ever_attached: false,
|
||||
first: Some(now),
|
||||
last: now,
|
||||
next_scan: now,
|
||||
retry_at: now,
|
||||
attach_retry: now,
|
||||
failures: 0,
|
||||
attached: false,
|
||||
error: None,
|
||||
watch_error: None,
|
||||
}
|
||||
}
|
||||
fn dirty(&mut self, paths: Option<&[PathBuf]>, now: Instant, media_hint: bool) {
|
||||
self.media_hint |= media_hint;
|
||||
self.first.get_or_insert(now);
|
||||
self.last = now;
|
||||
match paths {
|
||||
Some(paths) if !self.full => {
|
||||
self.paths.extend(paths.iter().cloned());
|
||||
if self.paths.len() > MAX_PATHS {
|
||||
self.paths.clear();
|
||||
self.full = true;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
self.paths.clear();
|
||||
self.full = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
struct State {
|
||||
roots: BTreeMap<PathBuf, Root>,
|
||||
stop: bool,
|
||||
}
|
||||
struct Shared {
|
||||
state: Mutex<State>,
|
||||
changed: Condvar,
|
||||
ignored: Vec<PathBuf>,
|
||||
timing: Timing,
|
||||
cancelled: Arc<AtomicBool>,
|
||||
}
|
||||
/// Own this service for the application lifetime. Dropping joins its worker after
|
||||
/// the currently executing reconciliation callback finishes.
|
||||
pub struct WatchService {
|
||||
shared: Arc<Shared>,
|
||||
worker: Mutex<Option<JoinHandle<()>>>,
|
||||
}
|
||||
impl WatchService {
|
||||
pub fn start(
|
||||
roots: Vec<PathBuf>,
|
||||
ignored: Vec<PathBuf>,
|
||||
callback: impl Fn(ScanRequest) -> ScanOutcome + Send + 'static,
|
||||
) -> Result<Self, String> {
|
||||
Self::start_timed(roots, ignored, callback, Timing::default())
|
||||
}
|
||||
fn start_timed(
|
||||
roots: Vec<PathBuf>,
|
||||
ignored: Vec<PathBuf>,
|
||||
callback: impl Fn(ScanRequest) -> ScanOutcome + Send + 'static,
|
||||
timing: Timing,
|
||||
) -> Result<Self, String> {
|
||||
validate(&roots)?;
|
||||
if ignored.len() > 128 || ignored.iter().any(|p| !p.is_absolute()) {
|
||||
return Err("Watcher exclusions must be bounded absolute paths.".into());
|
||||
}
|
||||
let shared = Arc::new(Shared {
|
||||
state: Mutex::new(State {
|
||||
roots: BTreeMap::new(),
|
||||
stop: false,
|
||||
}),
|
||||
changed: Condvar::new(),
|
||||
ignored,
|
||||
timing,
|
||||
cancelled: Arc::new(AtomicBool::new(false)),
|
||||
});
|
||||
let target = shared.clone();
|
||||
let worker = thread::Builder::new()
|
||||
.name("searchi-reconcile".into())
|
||||
.spawn(move || run(target, callback))
|
||||
.map_err(|e| e.to_string())?;
|
||||
let service = Self {
|
||||
shared,
|
||||
worker: Mutex::new(Some(worker)),
|
||||
};
|
||||
service.replace_roots(roots)?;
|
||||
Ok(service)
|
||||
}
|
||||
/// Stop and join only this owned worker. Call on application exit to break
|
||||
/// callback ownership of the desktop emitter. Cancellation waits for the
|
||||
/// current bounded parser operation to reach its next cooperative check.
|
||||
pub fn stop(&self) {
|
||||
self.shared.cancelled.store(true, Ordering::Release);
|
||||
if let Ok(mut state) = self.shared.state.lock() {
|
||||
state.stop = true;
|
||||
self.shared.changed.notify_all();
|
||||
}
|
||||
let worker = self.worker.lock().unwrap_or_else(|e| e.into_inner()).take();
|
||||
if let Some(worker) = worker {
|
||||
if worker.thread().id() != thread::current().id() {
|
||||
let _ = worker.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_root(&self, root: PathBuf) -> Result<(), String> {
|
||||
let mut s = self
|
||||
.shared
|
||||
.state
|
||||
.lock()
|
||||
.map_err(|_| "Watcher is unavailable.")?;
|
||||
if s.roots.contains_key(&root) {
|
||||
return Ok(());
|
||||
}
|
||||
if s.roots.len() >= MAX_ROOTS || !root.is_absolute() {
|
||||
return Err("Watcher folders must be absolute and limited to 64.".into());
|
||||
}
|
||||
s.roots.insert(root, Root::new(Instant::now()));
|
||||
self.shared.changed.notify_all();
|
||||
Ok(())
|
||||
}
|
||||
pub fn replace_roots(&self, roots: Vec<PathBuf>) -> Result<(), String> {
|
||||
validate(&roots)?;
|
||||
let mut s = self
|
||||
.shared
|
||||
.state
|
||||
.lock()
|
||||
.map_err(|_| "Watcher is unavailable.")?;
|
||||
s.roots.retain(|p, _| roots.contains(p));
|
||||
for p in roots {
|
||||
s.roots
|
||||
.entry(p)
|
||||
.or_insert_with(|| Root::new(Instant::now()));
|
||||
}
|
||||
self.shared.changed.notify_all();
|
||||
Ok(())
|
||||
}
|
||||
pub fn status(&self) -> Vec<WatchStatus> {
|
||||
self.shared
|
||||
.state
|
||||
.lock()
|
||||
.map(|s| {
|
||||
s.roots
|
||||
.iter()
|
||||
.map(|(p, r)| WatchStatus {
|
||||
root: p.clone(),
|
||||
attached: r.attached,
|
||||
pending: r.first.is_some(),
|
||||
error: r.error.clone().or_else(|| r.watch_error.clone()),
|
||||
watch_error: r.watch_error.clone(),
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
impl Drop for WatchService {
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
}
|
||||
}
|
||||
fn validate(roots: &[PathBuf]) -> Result<(), String> {
|
||||
if roots.len() > MAX_ROOTS || roots.iter().any(|p| !p.is_absolute()) {
|
||||
return Err("Watcher folders must be absolute and limited to 64.".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn excluded(path: &Path, shared: &Shared) -> bool {
|
||||
shared.ignored.iter().any(|p| path.starts_with(p))
|
||||
|| path.components().any(|c| {
|
||||
matches!(
|
||||
c.as_os_str().to_str(),
|
||||
Some(".git" | "node_modules" | "target")
|
||||
)
|
||||
})
|
||||
}
|
||||
fn hint(shared: &Shared, event: notify::Result<Event>) {
|
||||
let mut s = match shared.state.lock() {
|
||||
Ok(s) => s,
|
||||
Err(_) => return,
|
||||
};
|
||||
let now = Instant::now();
|
||||
match event {
|
||||
Ok(e)
|
||||
if !e.need_rescan()
|
||||
&& matches!(
|
||||
e.kind,
|
||||
EventKind::Access(_)
|
||||
| EventKind::Modify(notify::event::ModifyKind::Metadata(
|
||||
notify::event::MetadataKind::AccessTime
|
||||
))
|
||||
) => {}
|
||||
Ok(e) if !e.need_rescan() && !e.paths.is_empty() => {
|
||||
// Events can contain unbounded path vectors; consume a bounded prefix
|
||||
// and invalidate all roots if its representation overflowed.
|
||||
if e.paths.len() > MAX_PATHS {
|
||||
for r in s.roots.values_mut() {
|
||||
r.dirty(None, now, true);
|
||||
}
|
||||
} else {
|
||||
for (root, r) in &mut s.roots {
|
||||
let paths: Vec<_> = e
|
||||
.paths
|
||||
.iter()
|
||||
.filter(|p| p.starts_with(root.as_path()) && !excluded(p, shared))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !paths.is_empty() {
|
||||
r.dirty(Some(&paths), now, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
for r in s.roots.values_mut() {
|
||||
r.dirty(None, now, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.changed.notify_all();
|
||||
}
|
||||
fn run(shared: Arc<Shared>, callback: impl Fn(ScanRequest) -> ScanOutcome) {
|
||||
let events = shared.clone();
|
||||
let mut watcher = notify::recommended_watcher(move |event| hint(&events, event)).ok();
|
||||
let mut watched = BTreeSet::new();
|
||||
loop {
|
||||
let roots = {
|
||||
let s = shared.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if s.stop {
|
||||
break;
|
||||
}
|
||||
s.roots.keys().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
for removed in watched
|
||||
.iter()
|
||||
.filter(|p| !roots.contains(p))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
{
|
||||
if let Some(w) = &mut watcher {
|
||||
let _ = w.unwatch(&removed);
|
||||
}
|
||||
watched.remove(&removed);
|
||||
}
|
||||
let now = Instant::now();
|
||||
for root in &roots {
|
||||
if watched.contains(root) && root.is_dir() {
|
||||
continue;
|
||||
}
|
||||
if watched.remove(root) {
|
||||
if let Some(r) = shared
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.roots
|
||||
.get_mut(root)
|
||||
{
|
||||
r.attached = false;
|
||||
}
|
||||
if let Some(w) = &mut watcher {
|
||||
let _ = w.unwatch(root);
|
||||
}
|
||||
}
|
||||
let due = shared
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.roots
|
||||
.get(root)
|
||||
.is_some_and(|r| now >= r.attach_retry);
|
||||
if !due {
|
||||
continue;
|
||||
}
|
||||
if watcher.is_none() {
|
||||
let events = shared.clone();
|
||||
watcher = notify::recommended_watcher(move |e| hint(&events, e)).ok();
|
||||
}
|
||||
let result = match &mut watcher {
|
||||
Some(w) => w
|
||||
.watch(root, RecursiveMode::Recursive)
|
||||
.map_err(|e| e.to_string()),
|
||||
None => Err(
|
||||
"Native watcher is unavailable; periodic reconciliation remains enabled."
|
||||
.into(),
|
||||
),
|
||||
};
|
||||
let mut s = shared.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(r) = s.roots.get_mut(root) {
|
||||
r.attached = result.is_ok();
|
||||
r.attach_retry = if result.is_ok() {
|
||||
now
|
||||
} else {
|
||||
now + shared.timing.retry
|
||||
};
|
||||
if result.is_ok() {
|
||||
r.watch_error = None;
|
||||
r.dirty(None, now, r.ever_attached);
|
||||
r.ever_attached = true;
|
||||
watched.insert(root.clone());
|
||||
} else {
|
||||
r.watch_error = result.err().map(|e| e.chars().take(512).collect());
|
||||
}
|
||||
}
|
||||
}
|
||||
let request = {
|
||||
let mut s = shared.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if s.stop {
|
||||
break;
|
||||
}
|
||||
let now = Instant::now();
|
||||
for r in s.roots.values_mut() {
|
||||
if now >= r.next_scan {
|
||||
r.dirty(None, now, false);
|
||||
r.next_scan = now + shared.timing.periodic;
|
||||
}
|
||||
}
|
||||
s.roots.iter_mut().find_map(|(root, r)| {
|
||||
let due = r.first.is_some_and(|first| {
|
||||
now >= r.retry_at
|
||||
&& (now.duration_since(r.last) >= shared.timing.debounce
|
||||
|| now.duration_since(first) >= shared.timing.max_delay)
|
||||
});
|
||||
if !due {
|
||||
return None;
|
||||
}
|
||||
let paths = if r.full {
|
||||
None
|
||||
} else {
|
||||
Some(std::mem::take(&mut r.paths).into_iter().collect())
|
||||
};
|
||||
r.full = false;
|
||||
let media_hint = std::mem::take(&mut r.media_hint);
|
||||
r.first = None;
|
||||
Some(ScanRequest {
|
||||
root: root.clone(),
|
||||
paths,
|
||||
cancelled: shared.cancelled.clone(),
|
||||
media_hint,
|
||||
})
|
||||
})
|
||||
};
|
||||
if let Some(request) = request {
|
||||
let root = request.root.clone();
|
||||
let paths = request.paths.clone();
|
||||
let media_hint = request.media_hint;
|
||||
let result =
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| callback(request)))
|
||||
.unwrap_or_else(|_| {
|
||||
ScanOutcome::Failed(
|
||||
"Folder reconciliation failed unexpectedly; retrying.".into(),
|
||||
)
|
||||
});
|
||||
let mut s = shared.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(r) = s.roots.get_mut(&root) {
|
||||
let now = Instant::now();
|
||||
match result {
|
||||
ScanOutcome::Completed => {
|
||||
r.failures = 0;
|
||||
r.error = None;
|
||||
r.retry_at = now;
|
||||
}
|
||||
ScanOutcome::Busy => {
|
||||
r.dirty(paths.as_deref(), now, media_hint);
|
||||
r.retry_at = now + Duration::from_millis(250);
|
||||
}
|
||||
ScanOutcome::Failed(e) => {
|
||||
r.dirty(paths.as_deref(), now, media_hint);
|
||||
r.failures = r.failures.saturating_add(1);
|
||||
r.error = Some(e.chars().take(512).collect());
|
||||
r.retry_at = now
|
||||
+ shared
|
||||
.timing
|
||||
.retry
|
||||
.saturating_mul(1u32 << r.failures.saturating_sub(1).min(4))
|
||||
.min(Duration::from_secs(60));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let s = shared.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if s.stop {
|
||||
break;
|
||||
}
|
||||
let _ = shared.changed.wait_timeout(s, Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::mpsc;
|
||||
fn fast() -> Timing {
|
||||
Timing {
|
||||
debounce: Duration::from_millis(40),
|
||||
max_delay: Duration::from_millis(120),
|
||||
periodic: Duration::from_secs(30),
|
||||
retry: Duration::from_millis(20),
|
||||
}
|
||||
}
|
||||
fn receive(rx: &mpsc::Receiver<ScanRequest>) -> ScanRequest {
|
||||
rx.recv_timeout(Duration::from_secs(10))
|
||||
.expect("watch reconciliation")
|
||||
}
|
||||
#[test]
|
||||
fn native_windows_or_linux_create_update_rename_delete() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let service = WatchService::start_timed(
|
||||
vec![root.path().to_owned()],
|
||||
vec![],
|
||||
move |r| {
|
||||
tx.send(r).unwrap();
|
||||
ScanOutcome::Completed
|
||||
},
|
||||
fast(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(receive(&rx).paths.is_none());
|
||||
let first = root.path().join("first.txt");
|
||||
let second = root.path().join("second.txt");
|
||||
std::fs::write(&first, "one").unwrap();
|
||||
assert!(receive(&rx).paths.unwrap().contains(&first));
|
||||
std::fs::write(&first, "two").unwrap();
|
||||
assert!(receive(&rx).paths.unwrap().contains(&first));
|
||||
std::fs::rename(&first, &second).unwrap();
|
||||
let rename = receive(&rx).paths.unwrap();
|
||||
assert!(rename.contains(&first) || rename.contains(&second));
|
||||
std::fs::remove_file(&second).unwrap();
|
||||
assert!(receive(&rx).paths.unwrap().contains(&second));
|
||||
assert!(service.status()[0].attached);
|
||||
}
|
||||
#[test]
|
||||
fn overflow_rescan_and_events_during_scan_are_retained() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let service = WatchService::start_timed(
|
||||
vec![root.path().to_owned()],
|
||||
vec![],
|
||||
move |r| {
|
||||
tx.send(r).unwrap();
|
||||
ScanOutcome::Completed
|
||||
},
|
||||
fast(),
|
||||
)
|
||||
.unwrap();
|
||||
let _ = receive(&rx);
|
||||
let paths = (0..129)
|
||||
.map(|i| root.path().join(format!("{i}.txt")))
|
||||
.collect();
|
||||
hint(
|
||||
&service.shared,
|
||||
Ok(Event::new(EventKind::Any).add_path(root.path().join("a"))),
|
||||
);
|
||||
let mut e = Event::new(EventKind::Any);
|
||||
e.paths = paths;
|
||||
hint(&service.shared, Ok(e));
|
||||
let overflow = receive(&rx);
|
||||
assert!(overflow.paths.is_none());
|
||||
assert!(overflow.media_hint);
|
||||
}
|
||||
#[test]
|
||||
fn events_arriving_during_reconciliation_survive_completion() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let release_rx = Mutex::new(release_rx);
|
||||
let service = WatchService::start_timed(
|
||||
vec![root.path().to_owned()],
|
||||
vec![],
|
||||
move |r| {
|
||||
tx.send(r).unwrap();
|
||||
release_rx.lock().unwrap().recv().unwrap();
|
||||
ScanOutcome::Completed
|
||||
},
|
||||
fast(),
|
||||
)
|
||||
.unwrap();
|
||||
let _ = receive(&rx);
|
||||
let path = root.path().join("later.txt");
|
||||
hint(
|
||||
&service.shared,
|
||||
Ok(Event::new(EventKind::Any).add_path(path.clone())),
|
||||
);
|
||||
release_tx.send(()).unwrap();
|
||||
assert!(receive(&rx).paths.unwrap().contains(&path));
|
||||
release_tx.send(()).unwrap();
|
||||
drop(service);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn callback_panic_retries_and_recovers() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let attempts = AtomicBool::new(false);
|
||||
let service = WatchService::start_timed(
|
||||
vec![root.path().to_owned()],
|
||||
vec![],
|
||||
move |r| {
|
||||
if !attempts.swap(true, Ordering::AcqRel) {
|
||||
panic!("owned callback failure");
|
||||
}
|
||||
tx.send(r).unwrap();
|
||||
ScanOutcome::Completed
|
||||
},
|
||||
fast(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(receive(&rx).paths.is_none());
|
||||
service.stop();
|
||||
assert!(!service.status()[0].pending);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn busy_retry_preserves_native_media_hint() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let service = WatchService::start_timed(
|
||||
vec![root.path().to_owned()],
|
||||
vec![],
|
||||
move |r| {
|
||||
if r.media_hint {
|
||||
tx.send(r).unwrap();
|
||||
ScanOutcome::Busy
|
||||
} else {
|
||||
tx.send(r).unwrap();
|
||||
ScanOutcome::Completed
|
||||
}
|
||||
},
|
||||
fast(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!receive(&rx).media_hint);
|
||||
let mut event = Event::new(EventKind::Any);
|
||||
event.attrs.set_flag(notify::event::Flag::Rescan);
|
||||
hint(&service.shared, Ok(event));
|
||||
let first = receive(&rx);
|
||||
assert!(first.paths.is_none());
|
||||
assert!(first.media_hint);
|
||||
assert!(receive(&rx).media_hint);
|
||||
service.stop();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn periodic_reconciliation_and_shutdown_signal() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let mut timing = fast();
|
||||
timing.periodic = Duration::from_millis(150);
|
||||
let service = WatchService::start_timed(
|
||||
vec![root.path().to_owned()],
|
||||
vec![],
|
||||
move |r| {
|
||||
tx.send(r).unwrap();
|
||||
ScanOutcome::Completed
|
||||
},
|
||||
timing,
|
||||
)
|
||||
.unwrap();
|
||||
let initial = receive(&rx);
|
||||
assert!(initial.paths.is_none());
|
||||
assert!(!initial.media_hint);
|
||||
let periodic = receive(&rx);
|
||||
assert!(periodic.paths.is_none());
|
||||
assert!(!periodic.media_hint);
|
||||
drop(service);
|
||||
assert!(initial.cancelled.load(Ordering::Acquire));
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,13 @@ interface DesktopStatus {
|
||||
settings: Settings;
|
||||
indexing: boolean;
|
||||
shortcut_status: string;
|
||||
watching?: {
|
||||
root: string;
|
||||
attached: boolean;
|
||||
pending: boolean;
|
||||
error: string | null;
|
||||
watch_error: string | null;
|
||||
}[];
|
||||
}
|
||||
interface SemanticStatus {
|
||||
model: string;
|
||||
@@ -452,6 +459,16 @@ async function refreshDesktopStatus() {
|
||||
const path = document.createElement("span");
|
||||
path.textContent = displayPath(root);
|
||||
path.className = "path";
|
||||
const watched = status.watching?.find((entry) => entry.root === root);
|
||||
if (watched) {
|
||||
path.title = watched.error
|
||||
? `Automatic updates need attention: ${watched.error}`
|
||||
: watched.pending
|
||||
? "Checking for file changes…"
|
||||
: watched.attached
|
||||
? "Watching for file changes"
|
||||
: "Checking for changes automatically";
|
||||
}
|
||||
const reindex = document.createElement("button");
|
||||
reindex.type = "button";
|
||||
reindex.textContent = "Reindex";
|
||||
@@ -1039,6 +1056,31 @@ function submitSearch(request: SearchRequest, lane: SearchLane) {
|
||||
(lane === "lexical" ? lexicalLane : hybridLane).submit(request);
|
||||
}
|
||||
|
||||
function refreshIndexedSearch() {
|
||||
const previous = latestSearch;
|
||||
if (
|
||||
!previous ||
|
||||
previous.id !== requestId ||
|
||||
query.value.trim() !== previous.query
|
||||
)
|
||||
return;
|
||||
clearTimeout(hybridTimer);
|
||||
lexicalLane.clear();
|
||||
hybridLane.clear();
|
||||
querySemanticFailed = false;
|
||||
shownLane = "lexical";
|
||||
// Keep the original input/first-response timing while rejecting responses
|
||||
// captured before the index changed, even when the query text stays identical.
|
||||
const refreshed: SearchRequest = { ...previous, id: ++requestId };
|
||||
latestSearch = refreshed;
|
||||
submitSearch(refreshed, "lexical");
|
||||
if (canSearchSemantically()) {
|
||||
hybridTimer = setTimeout(() => {
|
||||
if (isCurrent(refreshed)) submitSearch(refreshed, "hybrid");
|
||||
}, SEMANTIC_SETTLE_MS);
|
||||
}
|
||||
}
|
||||
|
||||
const isCurrent = (request: SearchRequest) => request.id === requestId;
|
||||
const lexicalLane = new LatestLane<SearchRequest>(
|
||||
async (request) => {
|
||||
@@ -1366,6 +1408,19 @@ async function initializeSurface() {
|
||||
void refreshDesktopStatus().catch(showError);
|
||||
queueSemanticRefresh(0);
|
||||
}).catch(showError);
|
||||
if (isNativeDesktop()) {
|
||||
void listen("folder-indexing-changed", () => {
|
||||
void refreshDesktopStatus().catch(showError);
|
||||
queueSemanticRefresh(0);
|
||||
}).catch(showError);
|
||||
void listen("search-index-changed", () => {
|
||||
folderRevision++;
|
||||
void refreshStats().catch(showError);
|
||||
void refreshDesktopStatus().catch(showError);
|
||||
queueSemanticRefresh(0);
|
||||
refreshIndexedSearch();
|
||||
}).catch(showError);
|
||||
}
|
||||
// Poll only during a user-triggered job; idle indexing status does no work.
|
||||
setInterval(() => {
|
||||
if (indexingActive) void refreshDesktopStatus().catch(showError);
|
||||
|
||||
Reference in new issue
Block a user