Harden worker IPC and recoverable runtime staging
This commit is contained in:
1 parent
0bba1a0aa7
commit
b3ebd7a894
12 files changed
+1146
-69
No files matched your search
@@ -7,6 +7,10 @@ dist/
|
||||
.searchi/
|
||||
__pycache__/
|
||||
/apps/desktop/src-tauri/gen/
|
||||
/apps/desktop/src-tauri/.searchi-stage.lock
|
||||
/apps/desktop/src-tauri/.searchi-stage.json
|
||||
/apps/desktop/src-tauri/.resources.stage-*/
|
||||
/apps/desktop/src-tauri/.resources.backup-*/
|
||||
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
|
||||
/apps/desktop/src-tauri/resources/vulkan-1.dll
|
||||
/apps/desktop/src-tauri/resources/*.dll
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
//! Explicit data-directory isolation for portable use and native release QA.
|
||||
//! Ordinary launches continue to use the operating system's app-data directory.
|
||||
use std::{ffi::OsString, io, path::PathBuf};
|
||||
|
||||
pub fn resolve(default: PathBuf, override_path: Option<OsString>) -> io::Result<PathBuf> {
|
||||
let Some(value) = override_path else {
|
||||
return Ok(default);
|
||||
};
|
||||
let path = PathBuf::from(value);
|
||||
if !path.is_absolute() || path.as_os_str().len() > 32768 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"SEARCHI_DATA_DIR must be an absolute bounded directory path",
|
||||
));
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn explicit_directory_isolates_settings_and_index_without_changing_default() {
|
||||
let default = std::env::temp_dir().join("searchi-default");
|
||||
let isolated = std::env::temp_dir().join("searchi-isolated");
|
||||
assert_eq!(resolve(default.clone(), None).unwrap(), default);
|
||||
assert_eq!(
|
||||
resolve(default, Some(isolated.clone().into_os_string())).unwrap(),
|
||||
isolated
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_override_is_rejected_without_falling_back_to_user_data() {
|
||||
let default = std::env::temp_dir().join("searchi-default");
|
||||
for value in [OsString::new(), OsString::from("relative/data")] {
|
||||
assert_eq!(
|
||||
resolve(default.clone(), Some(value)).unwrap_err().kind(),
|
||||
io::ErrorKind::InvalidInput
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
#[cfg(all(not(debug_assertions), not(feature = "custom-protocol")))]
|
||||
compile_error!("Release desktop builds require --features custom-protocol to embed the frontend");
|
||||
|
||||
mod ai_commands;
|
||||
mod data_directory;
|
||||
mod model_download;
|
||||
mod semantic_commands;
|
||||
mod settings;
|
||||
@@ -394,7 +398,10 @@ fn main() {
|
||||
}
|
||||
}).build())
|
||||
.setup(|app| {
|
||||
let directory = app.path().app_data_dir()?;
|
||||
let directory = data_directory::resolve(
|
||||
app.path().app_data_dir()?,
|
||||
std::env::var_os("SEARCHI_DATA_DIR"),
|
||||
)?;
|
||||
std::fs::create_dir_all(&directory)?;
|
||||
let database = directory.join("searchi.sqlite");
|
||||
let lexical = Arc::new(Mutex::new(SearchIndex::open(&database)?));
|
||||
@@ -413,15 +420,15 @@ fn main() {
|
||||
let development_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..");
|
||||
let development_models = development_root.join(".searchi/models/embeddinggemma2");
|
||||
let bundled_models = app.path().resource_dir()?.join("embeddinggemma2");
|
||||
let model_directory = settings.value.model_directory.clone().unwrap_or_else(|| {
|
||||
if cfg!(debug_assertions) && development_models.is_dir() {
|
||||
let model_directory = if let Some(configured) = settings.value.model_directory.clone() {
|
||||
configured
|
||||
} else if cfg!(debug_assertions) && development_models.is_dir() {
|
||||
development_models.canonicalize().unwrap_or(development_models)
|
||||
} else if bundled_models.is_dir() {
|
||||
} else if searchi_assets::has_model_files(&bundled_models)? {
|
||||
bundled_models
|
||||
} else {
|
||||
searchi_assets::default_model_directory().unwrap_or_else(|_| directory.join("Models"))
|
||||
}
|
||||
});
|
||||
};
|
||||
let worker_name = if cfg!(windows) { "searchi-gemma-worker.exe" } else { "searchi-gemma-worker" };
|
||||
let development_worker = development_root.join(".searchi/runtime/cpu")
|
||||
.join(if cfg!(windows) { "Release" } else { "" }).join(worker_name);
|
||||
|
||||
@@ -94,6 +94,27 @@ pub fn default_model_directory() -> Result<PathBuf> {
|
||||
.join("embeddinggemma2"))
|
||||
}
|
||||
|
||||
/// Cheap location selection for optional bundled assets. This checks regular
|
||||
/// files and expected lengths only; `verify_cached` still verifies SHA-256
|
||||
/// before setup considers the model usable. Partial bundles use managed cache.
|
||||
pub fn has_model_files(directory: &Path) -> Result<bool> {
|
||||
model_files_present(directory, &manifest()?.files)
|
||||
}
|
||||
|
||||
fn model_files_present(directory: &Path, artifacts: &[Artifact]) -> Result<bool> {
|
||||
for artifact in artifacts {
|
||||
let metadata = match fs::metadata(directory.join(&artifact.path)) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => return Err(error).context("inspect bundled model artifact"),
|
||||
};
|
||||
if !metadata.is_file() || metadata.len() != artifact.bytes {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Offline validation includes file length and full SHA-256, not mere existence.
|
||||
pub fn verify_cached(directory: &Path) -> Result<bool> {
|
||||
for artifact in manifest()?.files {
|
||||
@@ -383,6 +404,32 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_bundle_requires_both_regular_files_at_expected_sizes() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let artifacts = [artifact("one.gguf", b"abc"), artifact("two.gguf", b"xyz")];
|
||||
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||
fs::write(directory.path().join("one.gguf"), b"abc").unwrap();
|
||||
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||
fs::create_dir(directory.path().join("two.gguf")).unwrap();
|
||||
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||
fs::remove_dir(directory.path().join("two.gguf")).unwrap();
|
||||
fs::write(directory.path().join("two.gguf"), b"xyz").unwrap();
|
||||
assert!(model_files_present(directory.path(), &artifacts).unwrap());
|
||||
fs::write(directory.path().join("one.gguf"), b"ab").unwrap();
|
||||
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundled_file_presence_never_substitutes_for_sha_verification() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let artifact = artifact("one.gguf", b"abc");
|
||||
let path = directory.path().join(&artifact.path);
|
||||
fs::write(&path, b"xyz").unwrap();
|
||||
assert!(model_files_present(directory.path(), std::slice::from_ref(&artifact)).unwrap());
|
||||
assert!(!verified(&path, &artifact).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_size_corruption_is_detected_and_missing_cache_is_offline() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -60,7 +60,10 @@ fn query_operation(info: &WorkerInfo, code: bool) -> Result<&'static str> {
|
||||
/// Persistent isolated native worker. No HTTP or remote model fallback.
|
||||
pub struct GemmaEmbedder {
|
||||
child: Child,
|
||||
stdin: ChildStdin,
|
||||
requests: Option<mpsc::SyncSender<Vec<u8>>>,
|
||||
writes: Receiver<std::result::Result<(), String>>,
|
||||
writer_thread: Option<thread::JoinHandle<()>>,
|
||||
reader_thread: Option<thread::JoinHandle<()>>,
|
||||
responses: Receiver<std::result::Result<Value, String>>,
|
||||
pub info: WorkerInfo,
|
||||
healthy: bool,
|
||||
@@ -73,24 +76,48 @@ pub struct GemmaEmbedder {
|
||||
|
||||
impl Drop for GemmaEmbedder {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
self.stop_worker();
|
||||
}
|
||||
}
|
||||
|
||||
type WriterTransport = (
|
||||
mpsc::SyncSender<Vec<u8>>,
|
||||
Receiver<std::result::Result<(), String>>,
|
||||
thread::JoinHandle<()>,
|
||||
);
|
||||
|
||||
fn spawn_writer(mut stdin: ChildStdin) -> WriterTransport {
|
||||
let (requests, pending) = mpsc::sync_channel::<Vec<u8>>(1);
|
||||
let (completed, writes) = mpsc::sync_channel(1);
|
||||
let writer = thread::spawn(move || {
|
||||
while let Ok(encoded) = pending.recv() {
|
||||
let result = stdin
|
||||
.write_all(&encoded)
|
||||
.and_then(|_| stdin.write_all(b"\n"))
|
||||
.and_then(|_| stdin.flush())
|
||||
.map_err(|error| format!("Cannot write local Gemma worker request: {error}"));
|
||||
let failed = result.is_err();
|
||||
if completed.try_send(result).is_err() || failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
(requests, writes, writer)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct StartupTransportFailure(String);
|
||||
impl std::fmt::Display for StartupTransportFailure {
|
||||
struct WorkerTransportFailure(String);
|
||||
impl std::fmt::Display for WorkerTransportFailure {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for StartupTransportFailure {}
|
||||
impl std::error::Error for WorkerTransportFailure {}
|
||||
|
||||
/// Retry only process/transport failure, never an incompatible protocol or model.
|
||||
fn start_with_cpu_fallback<T>(cpu: bool, mut start: impl FnMut(bool) -> Result<T>) -> Result<T> {
|
||||
match start(cpu) {
|
||||
Err(error) if !cpu && error.downcast_ref::<StartupTransportFailure>().is_some() => {
|
||||
Err(error) if !cpu && error.downcast_ref::<WorkerTransportFailure>().is_some() => {
|
||||
start(true).with_context(|| {
|
||||
format!("CPU initialization failed after automatic worker startup failure: {error}")
|
||||
})
|
||||
@@ -163,7 +190,7 @@ impl GemmaEmbedder {
|
||||
command.creation_flags(0x08000000); // CREATE_NO_WINDOW
|
||||
}
|
||||
let mut child = command.spawn().map_err(|error| {
|
||||
StartupTransportFailure(format!("Cannot start local Gemma worker: {error}"))
|
||||
WorkerTransportFailure(format!("Cannot start local Gemma worker: {error}"))
|
||||
})?;
|
||||
let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else {
|
||||
let _ = child.kill();
|
||||
@@ -171,12 +198,12 @@ impl GemmaEmbedder {
|
||||
anyhow::bail!("Gemma worker protocol pipes missing");
|
||||
};
|
||||
let (sender, responses) = mpsc::sync_channel(1);
|
||||
thread::spawn(move || {
|
||||
let reader_thread = thread::spawn(move || {
|
||||
let mut reader = BufReader::new(stdout);
|
||||
loop {
|
||||
let response = read_response(&mut reader);
|
||||
let failed = response.is_err();
|
||||
if sender.send(response).is_err() || failed {
|
||||
if sender.try_send(response).is_err() || failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -186,13 +213,14 @@ impl GemmaEmbedder {
|
||||
other => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
let _ = reader_thread.join();
|
||||
// Malformed/oversized JSON is a protocol violation, not evidence
|
||||
// that hardware acceleration failed. Do not mask it with retry.
|
||||
if matches!(&other, Ok(Err(error)) if error.starts_with("Invalid Gemma worker JSON:") || error == "Gemma worker response exceeds protocol byte budget")
|
||||
{
|
||||
anyhow::bail!("Invalid Gemma worker startup protocol: {other:?}");
|
||||
}
|
||||
return Err(StartupTransportFailure(format!(
|
||||
return Err(WorkerTransportFailure(format!(
|
||||
"Gemma worker did not initialize: {other:?}"
|
||||
))
|
||||
.into());
|
||||
@@ -203,6 +231,7 @@ impl GemmaEmbedder {
|
||||
Err(error) => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
let _ = reader_thread.join();
|
||||
return Err(error).context("Invalid Gemma worker handshake");
|
||||
}
|
||||
};
|
||||
@@ -216,11 +245,16 @@ impl GemmaEmbedder {
|
||||
{
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
let _ = reader_thread.join();
|
||||
anyhow::bail!("Gemma worker handshake violates configured model/device contract");
|
||||
}
|
||||
let (requests, writes, writer_thread) = spawn_writer(stdin);
|
||||
Ok(Self {
|
||||
child,
|
||||
stdin,
|
||||
requests: Some(requests),
|
||||
writes,
|
||||
writer_thread: Some(writer_thread),
|
||||
reader_thread: Some(reader_thread),
|
||||
responses,
|
||||
info,
|
||||
healthy: true,
|
||||
@@ -246,7 +280,10 @@ impl GemmaEmbedder {
|
||||
);
|
||||
let response = match self.exchange(&encoded) {
|
||||
Ok(response) => response,
|
||||
Err(error) if can_retry_cpu(&self.info.device, self.force_cpu) => {
|
||||
Err(error)
|
||||
if can_retry_cpu(&self.info.device, self.force_cpu)
|
||||
&& error.downcast_ref::<WorkerTransportFailure>().is_some() =>
|
||||
{
|
||||
// GPU runtimes can abort rather than return an encoder error.
|
||||
// Restart once as CPU and retry this same bounded request.
|
||||
let _ = self.child.wait();
|
||||
@@ -296,33 +333,76 @@ impl GemmaEmbedder {
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
|
||||
let write_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||
let sent = (|| {
|
||||
self.stdin.write_all(encoded)?;
|
||||
self.stdin.write_all(b"\n")?;
|
||||
self.stdin.flush()
|
||||
})()
|
||||
.context("Cannot write local Gemma worker request");
|
||||
self.record_query_duration("request_write_us", write_started);
|
||||
if let Err(error) = sent {
|
||||
self.healthy = false;
|
||||
self.info.ready = false;
|
||||
fn stop_worker(&mut self) {
|
||||
self.requests.take();
|
||||
// The shipped native worker does not spawn descendants. Killing/reaping
|
||||
// it closes its pipe ends and releases any blocked writer/read thread.
|
||||
let _ = self.child.kill();
|
||||
return Err(error);
|
||||
let _ = self.child.wait();
|
||||
if let Some(writer) = self.writer_thread.take() {
|
||||
let _ = writer.join();
|
||||
}
|
||||
let wait_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||
let response = match self.responses.recv_timeout(Duration::from_secs(180)) {
|
||||
Ok(Ok(value)) => value,
|
||||
if let Some(reader) = self.reader_thread.take() {
|
||||
let _ = reader.join();
|
||||
}
|
||||
}
|
||||
|
||||
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
|
||||
self.exchange_with_timeout(encoded, Duration::from_secs(180))
|
||||
}
|
||||
|
||||
fn exchange_with_timeout(&mut self, encoded: &[u8], timeout: Duration) -> Result<Value> {
|
||||
let deadline = Instant::now() + timeout;
|
||||
let write_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||
let result = (|| {
|
||||
self.requests
|
||||
.as_ref()
|
||||
.ok_or_else(|| WorkerTransportFailure("Gemma worker writer stopped".into()))?
|
||||
.try_send(encoded.to_vec())
|
||||
.map_err(|_| {
|
||||
WorkerTransportFailure("Gemma worker request queue unavailable".into())
|
||||
})?;
|
||||
match self
|
||||
.writes
|
||||
.recv_timeout(deadline.saturating_duration_since(Instant::now()))
|
||||
{
|
||||
Ok(Ok(())) => (),
|
||||
other => {
|
||||
self.healthy = false;
|
||||
self.info.ready = false;
|
||||
let _ = self.child.kill();
|
||||
anyhow::bail!("Gemma worker request failed: {other:?}");
|
||||
return Err(WorkerTransportFailure(format!(
|
||||
"Gemma worker request write failed or timed out: {other:?}"
|
||||
))
|
||||
.into())
|
||||
}
|
||||
}
|
||||
self.record_query_duration("request_write_us", write_started);
|
||||
let wait_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||
let response = match self
|
||||
.responses
|
||||
.recv_timeout(deadline.saturating_duration_since(Instant::now()))
|
||||
{
|
||||
Ok(Ok(value)) => value,
|
||||
Ok(Err(error))
|
||||
if error.starts_with("Invalid Gemma worker JSON:")
|
||||
|| error == "Gemma worker response exceeds protocol byte budget" =>
|
||||
{
|
||||
anyhow::bail!("Invalid Gemma worker response protocol: {error}")
|
||||
}
|
||||
other => {
|
||||
return Err(WorkerTransportFailure(format!(
|
||||
"Gemma worker request failed or timed out: {other:?}"
|
||||
))
|
||||
.into())
|
||||
}
|
||||
};
|
||||
self.record_query_duration("worker_wait_and_response_decode_us", wait_started);
|
||||
Ok(response)
|
||||
})();
|
||||
if result.is_err() {
|
||||
self.healthy = false;
|
||||
self.info.ready = false;
|
||||
self.stop_worker();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>> {
|
||||
@@ -710,6 +790,122 @@ fn verify_artifact(path: &Path, bytes: u64, sha: &str) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
// This helper is an inert pipe-protocol fixture, never an embedding provider.
|
||||
fn stalled_protocol_helper(read_request: bool) -> GemmaEmbedder {
|
||||
#[cfg(windows)]
|
||||
let mut command = {
|
||||
use std::os::windows::process::CommandExt;
|
||||
let mut command = Command::new("powershell.exe");
|
||||
command.args([
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
if read_request {
|
||||
"[Console]::In.ReadLine() | Out-Null; Start-Sleep -Seconds 20"
|
||||
} else {
|
||||
"Start-Sleep -Seconds 20"
|
||||
},
|
||||
]);
|
||||
command.creation_flags(0x08000000);
|
||||
command
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
let mut command = {
|
||||
let mut command = Command::new("sh");
|
||||
command.args([
|
||||
"-c",
|
||||
if read_request {
|
||||
"read line; exec sleep 20"
|
||||
} else {
|
||||
"exec sleep 20"
|
||||
},
|
||||
]);
|
||||
command
|
||||
};
|
||||
let mut child = command
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let (requests, writes, writer_thread) = spawn_writer(child.stdin.take().unwrap());
|
||||
let stdout = child.stdout.take().unwrap();
|
||||
let (sender, responses) = mpsc::sync_channel(1);
|
||||
let reader_thread = thread::spawn(move || {
|
||||
let response = read_response(&mut BufReader::new(stdout));
|
||||
let _ = sender.try_send(response);
|
||||
});
|
||||
GemmaEmbedder {
|
||||
child,
|
||||
requests: Some(requests),
|
||||
writes,
|
||||
writer_thread: Some(writer_thread),
|
||||
reader_thread: Some(reader_thread),
|
||||
responses,
|
||||
info: WorkerInfo {
|
||||
ready: true,
|
||||
model: "pipe protocol fixture".into(),
|
||||
dimensions: 0,
|
||||
device: "cpu".into(),
|
||||
threads: 1,
|
||||
runtime: "fixture".into(),
|
||||
media_supported: false,
|
||||
code_query_supported: false,
|
||||
},
|
||||
healthy: true,
|
||||
force_cpu: true,
|
||||
model_dir: PathBuf::new(),
|
||||
worker_path: PathBuf::new(),
|
||||
projector_verified: false,
|
||||
query_profile: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disconnected_writer_queue_is_transport_failure_and_reaps_helper() {
|
||||
let mut fixture = stalled_protocol_helper(false);
|
||||
let (sender, receiver) = mpsc::sync_channel(1);
|
||||
drop(receiver);
|
||||
fixture.requests = Some(sender);
|
||||
let error = fixture
|
||||
.exchange_with_timeout(b"{}", Duration::from_millis(300))
|
||||
.unwrap_err();
|
||||
assert!(error.downcast_ref::<WorkerTransportFailure>().is_some());
|
||||
assert!(fixture.child.try_wait().unwrap().is_some());
|
||||
assert!(!fixture.healthy && fixture.requests.is_none());
|
||||
assert!(fixture.writer_thread.is_none() && fixture.reader_thread.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entire_request_deadline_kills_and_joins_worker_that_never_reads_stdin() {
|
||||
let mut fixture = stalled_protocol_helper(false);
|
||||
let started = Instant::now();
|
||||
let error = fixture
|
||||
.exchange_with_timeout(&vec![b'x'; MAX_REQUEST - 1], Duration::from_millis(300))
|
||||
.unwrap_err();
|
||||
assert!(error.downcast_ref::<WorkerTransportFailure>().is_some());
|
||||
assert!(error.to_string().contains("write failed or timed out"));
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
assert!(fixture.child.try_wait().unwrap().is_some());
|
||||
assert!(!fixture.healthy && !fixture.info.ready);
|
||||
assert!(fixture.writer_thread.is_none() && fixture.reader_thread.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_wait_uses_same_deadline_and_reaps_silent_reader() {
|
||||
let mut fixture = stalled_protocol_helper(true);
|
||||
let started = Instant::now();
|
||||
let error = fixture
|
||||
.exchange_with_timeout(b"{}", Duration::from_millis(300))
|
||||
.unwrap_err();
|
||||
assert!(error.downcast_ref::<WorkerTransportFailure>().is_some());
|
||||
assert!(error.to_string().contains("request failed or timed out"));
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
assert!(fixture.child.try_wait().unwrap().is_some());
|
||||
assert!(fixture.requests.is_none());
|
||||
assert!(fixture.writer_thread.is_none() && fixture.reader_thread.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_startup_transport_failure_retries_cpu_once() {
|
||||
let mut attempts = Vec::new();
|
||||
@@ -718,7 +914,7 @@ mod tests {
|
||||
if cpu {
|
||||
Ok("cpu ready")
|
||||
} else {
|
||||
Err(StartupTransportFailure("worker closed its output before ready".into()).into())
|
||||
Err(WorkerTransportFailure("worker closed its output before ready".into()).into())
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
@@ -732,7 +928,7 @@ mod tests {
|
||||
let mut attempts = Vec::new();
|
||||
let result: Result<()> = start_with_cpu_fallback(cpu, |selected_cpu| {
|
||||
attempts.push(selected_cpu);
|
||||
Err(StartupTransportFailure("worker exited".into()).into())
|
||||
Err(WorkerTransportFailure("worker exited".into()).into())
|
||||
});
|
||||
let error = format!("{:#}", result.unwrap_err());
|
||||
assert_eq!(attempts, if cpu { vec![true] } else { vec![false, true] });
|
||||
|
||||
@@ -2,6 +2,17 @@
|
||||
|
||||
final result: browser captures checked; native desktop visual gate remains open
|
||||
|
||||
## Native release startup evidence (2026-10-12)
|
||||
|
||||
The extracted embedded-frontend Windows Release archive launches outside the
|
||||
checkout with isolated validation data and CPU-only inference/rendering.
|
||||
Accessibility confirms the `http://tauri.localhost/` frontend, one indexed file,
|
||||
one extracted content segment and ready Google semantic search using CPU.
|
||||
This verifies startup and engine status, not search interaction or appearance.
|
||||
Native input fails with `GetCursorPos 0x80070005`; capture fails with
|
||||
`CreateForMonitor 0x80070057`. No native screenshot was obtained. Native focus,
|
||||
keyboard navigation, transitions and visual inspection remain open.
|
||||
|
||||
## Browser-rendered implementation evidence (2026-10-12)
|
||||
|
||||
Four 1440×1000 captures now show the production frontend against the actual
|
||||
|
||||
+42
-7
@@ -35,8 +35,13 @@ cargo test -p searchi-service --test google_cpu -- --ignored --nocapture
|
||||
cargo test -p searchi-service --lib google_cpu_preserves -- --ignored --nocapture
|
||||
```
|
||||
|
||||
The worker is statically linked with the MSVC CRT on Windows. Staging copies
|
||||
it and the llama.cpp license into the desktop resources directory. Add
|
||||
The explicit internal CPU worker uses the static MSVC CRT on Windows; normal
|
||||
auto builds use the shared runtime described below. Staging verifies a complete
|
||||
candidate resource directory, then swaps it into place under a process lock.
|
||||
Its recovery journal restores an interrupted swap on the next invocation;
|
||||
failed restoration retains the previous backup. Existing unrelated fonts and
|
||||
media resources are preserved. Fault-injected process interruption is tested;
|
||||
power-loss durability is not certified. Add
|
||||
`--bundle-model` to staging for an offline model bundle after downloading;
|
||||
otherwise desktop setup provisions verified models on first launch. Search
|
||||
itself performs no network inference or downloads. The worker now uses the verified media projector for genuine image/audio embeddings. The FFmpeg script stages verified standalone decoders and their notices.
|
||||
@@ -59,25 +64,55 @@ supports `--backend auto|metal|vulkan|cuda` builds with CPU fallback; do not
|
||||
exercise those acceleration backends in this VM. Their execution and failure
|
||||
recovery require validation on the corresponding release hardware.
|
||||
|
||||
### Isolated native desktop validation
|
||||
|
||||
Set `SEARCHI_DATA_DIR` to an absolute directory to isolate the desktop's
|
||||
settings and index for release QA or portable use. Normal launches use the OS
|
||||
app-data directory. Invalid overrides fail startup instead of falling back to
|
||||
the user's regular data. Model downloads still use the configured model
|
||||
directory or managed cache, so isolated tests should also configure a verified
|
||||
fixture model directory in their own settings.
|
||||
|
||||
```powershell
|
||||
$env:SEARCHI_DATA_DIR = 'C:\searchi-qa-data'
|
||||
$env:SEARCHI_FORCE_CPU = '1'
|
||||
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
|
||||
```
|
||||
|
||||
Extract and launch the entire Release archive outside the source checkout.
|
||||
Direct Cargo release builds must embed the frontend:
|
||||
|
||||
```powershell
|
||||
cargo build --manifest-path apps/desktop/src-tauri/Cargo.toml --release --features custom-protocol --locked
|
||||
```
|
||||
|
||||
The crate rejects Release compilation without this feature; the Tauri build
|
||||
CLI enables it automatically. A development-server URL is not a release gate.
|
||||
Record native window interaction separately from process startup, browser
|
||||
screenshots and worker inference; those are different release gates.
|
||||
|
||||
## Headless Linux Google search
|
||||
|
||||
These are the current script and CLI interfaces, not evidence of an executed
|
||||
Linux release. On a native Linux build host, run from the repository root:
|
||||
Linux release. On a native Linux build host with Vulkan development libraries
|
||||
and `glslc`, run from the repository root:
|
||||
|
||||
```sh
|
||||
python3 tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||
python3 tools/embeddinggemma2/build_runtime.py --backend auto --jobs 2
|
||||
python3 tools/embeddinggemma2/download.py
|
||||
python3 tools/media-runtime/provision.py
|
||||
export SEARCHI_FORCE_CPU=1
|
||||
cargo build --release -p searchi-cli --locked
|
||||
./target/release/searchi index --db searchi.sqlite --root /path/to/documents
|
||||
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
|
||||
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
|
||||
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/auto/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
|
||||
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/auto/searchi-gemma-worker --cpu-threads 1
|
||||
```
|
||||
|
||||
`embed` builds document, image, video and audio/music vectors by default. Use
|
||||
`--text-only` explicitly to omit media. Default media indexing needs paired
|
||||
FFmpeg/FFprobe paths or packaged helpers beside the installed CLI.
|
||||
This normal worker detects usable acceleration and falls back to CPU. Internal
|
||||
CPU-only model tests may use `--backend cpu` and `SEARCHI_FORCE_CPU=1`; those
|
||||
test artifacts are not a separate distribution edition.
|
||||
|
||||
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
|
||||
command. It provisions model assets, not the native worker. Package the worker
|
||||
|
||||
+16
-6
@@ -67,11 +67,16 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
- [x] Windows desktop archive packager includes explicit runtime/helper files
|
||||
and notices, checks architecture and records per-file checksums; excludes
|
||||
model caches and user data. Controlled packaging regression tests pass.
|
||||
- [ ] Verify embedded-frontend Release archive launch outside the source tree,
|
||||
clean-machine prerequisites, GPU dependencies, upgrades and signing.
|
||||
- [x] Embedded-frontend Windows Release archive launches outside the source tree
|
||||
with isolated data and a ready real Google CPU semantic generation.
|
||||
- [ ] Verify native interactions, clean-machine prerequisites, GPU dependencies,
|
||||
upgrades and signing.
|
||||
- [x] Preflight native license and the complete offline Google model pair before
|
||||
modifying staged resources; corrupted/missing assets preserve existing worker.
|
||||
- [ ] Atomic resource staging and interrupted-copy recovery for release bundles.
|
||||
- [x] Transactional resource staging, verified copied bytes, process locking,
|
||||
interrupted-copy/swap recovery and backup preservation on failed rollback.
|
||||
- [ ] Verify power-loss durability and release-tool interaction with resource
|
||||
directory swaps on each OS; fault-injected process interruption is narrower.
|
||||
- [x] Shared monotonic Basic/Debug search metrics, granular bounded engine traces,
|
||||
actual worker query diagnostics, and headless JSON performance output.
|
||||
- [ ] Verify detailed performance UI interaction on native desktop platforms;
|
||||
@@ -84,7 +89,9 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
Warm p95 alone does not satisfy the first-use requirement.
|
||||
- [ ] Test migrations, interrupted work, invalid documents, inaccessible/offline roots and cloud placeholders.
|
||||
- [ ] Automated native build matrix and documented supported cross-compilation paths.
|
||||
- [ ] Execute Linux/macOS platform tests; create Windows/macOS/Linux installers.
|
||||
- [x] Native Linux headless workspace tests: 93 passed, 11 explicitly ignored.
|
||||
- [ ] Execute Linux native Google worker and desktop/macOS platform tests;
|
||||
create Windows/macOS/Linux installers.
|
||||
- [ ] Signing/notarization configuration, upgrade/rollback validation, release instructions.
|
||||
|
||||
## 6. Local agents and optional answers
|
||||
@@ -115,8 +122,11 @@ Audit and measured priorities: [EMBEDDINGGEMMA2-AUDIT.md](EMBEDDINGGEMMA2-AUDIT.
|
||||
preserve filtering, source freshness, failures and actual warmup probes.
|
||||
- [x] Reduce semantic settle delay to 120 ms and retain old ready generation
|
||||
during rebuilding; test initial-generation and activation readiness boundaries.
|
||||
- [ ] Bound whole IPC write/read deadlines and recover from a nonreading helper.
|
||||
- [ ] Declare/test x86 CPU minimum or baseline/AVX2 runtime selection.
|
||||
- [x] Bound whole IPC write/read deadlines and recover from a nonreading helper;
|
||||
CPU-only real helper-process regressions cover blocked writes and responses.
|
||||
- [x] Windows x64 baseline/optimized CPU runtime selection and OS AVX-state
|
||||
checks, verified with real packaged baseline and optimized inference.
|
||||
- [ ] Validate CPU selection on older hardware and remaining architectures.
|
||||
- [ ] Incrementally reuse unchanged document/media vectors in atomic generations.
|
||||
- [ ] Benchmark genuine token-budgeted multi-sequence batching and isolation.
|
||||
- [ ] Implement and measure shared foreground/indexer resource budgets/priority.
|
||||
|
||||
@@ -20,6 +20,12 @@ under the opt-in minilm-proof feature and tools/semantic-proof/HISTORICAL.md.
|
||||
The persistent native helper uses bounded NDJSON over stdin/stdout. It has no
|
||||
HTTP server or download code. Rust verifies the text model's length and SHA256
|
||||
before starting it. Setup provisions assets; search never downloads them.
|
||||
Each request has one 180-second IPC deadline across pipe writing and response
|
||||
reading. Single-slot queues bound pending work. A stalled write or response
|
||||
kills/reaps the helper and joins both IPC threads; only transport failures
|
||||
qualify for a bounded CPU restart. Malformed/oversized responses remain
|
||||
protocol errors. Cleanup assumes the shipped helper spawns no descendants
|
||||
holding its pipe handles, matching the native implementation.
|
||||
|
||||
Production device mode is auto, with available compiled acceleration and CPU
|
||||
fallback. Model/context initialization and returned inference/nonfinite-output
|
||||
|
||||
@@ -7,6 +7,39 @@ and `SEARCHI_FORCE_CPU=1`.
|
||||
|
||||
## Current integration checkpoint (2026-10-12)
|
||||
|
||||
The reliability checkpoint passes 93 workspace tests on Windows and 93 on
|
||||
native Linux (Ubuntu 24.04 under WSL); each explicitly ignores 11 optional
|
||||
integration tests. Linux uses its native Rust toolchain and an isolated ext4
|
||||
build cache. This verifies headless core portability, not a Linux desktop,
|
||||
Linux native Google worker, GPU or installer. Windows desktop passes 17 tests;
|
||||
strict workspace and desktop Clippy checks pass. Five genuine Google model
|
||||
integration tests pass in forced-CPU mode.
|
||||
|
||||
Worker IPC now applies one deadline across sending and receiving, with bounded
|
||||
queues and owned writer/reader threads. Transport failures terminate and reap
|
||||
the worker; eligible GPU transport failures retry on CPU. Invalid protocol
|
||||
responses remain errors. Fifteen semantic unit tests pass, including a blocked
|
||||
pipe and disconnected writer; one optional integration test is ignored.
|
||||
|
||||
Resource staging passes 24 tests on Windows (one symlink-permission skip) and
|
||||
23 on Linux (two Windows-only skips). Locking, recovery, rollback preservation,
|
||||
link rejection and concurrent edits before the final swap are covered. Actual
|
||||
automatic-backend staging also succeeds. These are process-failure checks;
|
||||
power-loss durability and concurrent arbitrary writes after the final check
|
||||
remain release gates.
|
||||
|
||||
The latest unsigned Windows archive launches outside the checkout with an
|
||||
embedded frontend at `http://tauri.localhost/`, isolated data, one real indexed
|
||||
document and a ready Google CPU semantic generation. The archive is
|
||||
145,217,175 bytes, SHA-256
|
||||
`fb454a9dbf258f51044796ab24a5942a8f90804d0824ca0e2dc5f59b7847ec3b`.
|
||||
Native accessibility confirms startup and status only. Input and screenshot
|
||||
capture remain denied by the environment; native interaction, appearance,
|
||||
clean-machine installation and GPU behavior are unverified. Release builds
|
||||
without the embedded-frontend feature now fail explicitly.
|
||||
|
||||
The following package-dispatch evidence predates this reliability checkpoint:
|
||||
|
||||
The normal Windows x64 package now includes baseline and optimized CPU DLLs
|
||||
with CPU/OS feature checks, plus the Vulkan backend. Shared DLLs use a common
|
||||
MSVC CRT: a first `/MT` prototype crashed because model file handles crossed
|
||||
|
||||
@@ -1,16 +1,327 @@
|
||||
"""Stage the native worker for installers; optional offline model bundle."""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
|
||||
|
||||
|
||||
def _is_link(path):
|
||||
"""Reject Windows reparse points, including junctions, on Python 3.11+."""
|
||||
path = Path(path)
|
||||
if path.is_symlink():
|
||||
return True
|
||||
try:
|
||||
attributes = getattr(path.lstat(), "st_file_attributes", 0)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return bool(attributes & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
|
||||
|
||||
|
||||
def _checked_path(path, parent):
|
||||
"""Validate absolute workspace boundaries before any move or recursive delete."""
|
||||
path, parent = Path(path).absolute(), Path(parent).resolve()
|
||||
if _is_link(path) or path.parent.resolve() != parent:
|
||||
raise RuntimeError("Staging path escapes its checked parent or is a link")
|
||||
if path.exists() and path.resolve() != parent / path.name:
|
||||
raise RuntimeError("Staging path resolves outside its checked parent")
|
||||
return path
|
||||
|
||||
|
||||
def _check_tree(directory):
|
||||
if _is_link(directory):
|
||||
raise RuntimeError("Resource staging refuses symlinks or junctions")
|
||||
count = 0
|
||||
for parent, directories, files in os.walk(directory, followlinks=False):
|
||||
for name in [*directories, *files]:
|
||||
count += 1
|
||||
if count > 100000:
|
||||
raise RuntimeError("Resource staging exceeds its file-count limit")
|
||||
path = Path(parent) / name
|
||||
if _is_link(path) or not path.resolve().is_relative_to(directory.resolve()):
|
||||
raise RuntimeError("Resource staging refuses symlinks or junctions")
|
||||
|
||||
|
||||
def _remove_tree(path, parent):
|
||||
checked = _checked_path(path, parent)
|
||||
if checked.exists():
|
||||
_check_tree(checked)
|
||||
shutil.rmtree(checked)
|
||||
|
||||
|
||||
def _move_directory(source, destination, parent):
|
||||
"""Bounded retry for transient Windows sharing locks during directory swaps."""
|
||||
for attempt in range(4):
|
||||
source = _checked_path(source, parent)
|
||||
destination = _checked_path(destination, parent)
|
||||
try:
|
||||
os.replace(source, destination)
|
||||
return
|
||||
except PermissionError:
|
||||
if attempt == 3:
|
||||
raise
|
||||
time.sleep(0.05 * (2 ** attempt))
|
||||
|
||||
|
||||
def _record(path):
|
||||
path = Path(path)
|
||||
if _is_link(path) or path.absolute() != path.resolve() or not path.is_file():
|
||||
raise RuntimeError(f"Staging source is missing or is a link: {path.name}")
|
||||
with path.open("rb") as stream:
|
||||
return path.stat().st_size, hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def _verified_copy(source, destination, expected=None):
|
||||
source, destination = Path(source), Path(destination)
|
||||
expected = expected or _record(source)
|
||||
if destination.absolute() != destination.resolve():
|
||||
raise RuntimeError("Staging copy target resolves through a link")
|
||||
shutil.copy2(source, destination)
|
||||
if _record(destination) != expected:
|
||||
raise RuntimeError(f"Staging source changed or copy was incomplete: {source.name}")
|
||||
return str(destination)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def staging_lock(parent):
|
||||
"""Kernel-held lock releases after process crashes; PID is informational."""
|
||||
path = _checked_path(parent / ".searchi-stage.lock", parent)
|
||||
with path.open("a+b") as stream:
|
||||
stream.seek(0, os.SEEK_END)
|
||||
if stream.tell() == 0:
|
||||
stream.write(b"\0")
|
||||
stream.flush()
|
||||
stream.seek(0)
|
||||
try:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as error:
|
||||
raise RuntimeError("Another resource staging process holds the lock") from error
|
||||
try:
|
||||
stream.seek(1)
|
||||
stream.truncate()
|
||||
stream.write(f"pid={os.getpid()}".encode())
|
||||
stream.flush()
|
||||
yield
|
||||
finally:
|
||||
stream.seek(0)
|
||||
if os.name == "nt":
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _journal_paths(destination, state):
|
||||
if (not isinstance(state, dict) or state.get("schema") != 1
|
||||
or state.get("owner") != "searchi-runtime-staging"
|
||||
or not isinstance(state.get("transaction"), str)
|
||||
or not re.fullmatch(r"[0-9a-f]{32}", state["transaction"])
|
||||
or type(state.get("had_destination")) is not bool):
|
||||
raise RuntimeError("Invalid resource staging recovery journal")
|
||||
parent = destination.parent.resolve()
|
||||
transaction = state["transaction"]
|
||||
return (_checked_path(parent / f".resources.stage-{transaction}", parent),
|
||||
_checked_path(parent / f".resources.backup-{transaction}", parent))
|
||||
|
||||
|
||||
def recover_staging(destination):
|
||||
"""Recover interrupted copy/swap while caller holds the staging lock.
|
||||
|
||||
This handles process interruption, not certified power-loss durability.
|
||||
An unsuccessful restoration retains the journal and original backup.
|
||||
"""
|
||||
parent = destination.parent.resolve()
|
||||
destination = _checked_path(destination, parent)
|
||||
journal = _checked_path(parent / ".searchi-stage.json", parent)
|
||||
if not journal.exists():
|
||||
return
|
||||
if journal.stat().st_size > 4096:
|
||||
raise RuntimeError("Oversized resource staging recovery journal")
|
||||
state = json.loads(journal.read_text(encoding="utf-8"))
|
||||
candidate, backup = _journal_paths(destination, state)
|
||||
for directory in [destination, candidate, backup]:
|
||||
if directory.exists():
|
||||
_check_tree(directory)
|
||||
marker = ".searchi-stage-complete"
|
||||
|
||||
def complete(directory):
|
||||
path = directory / marker
|
||||
return path.is_file() and path.stat().st_size == 32 and path.read_text() == state["transaction"]
|
||||
|
||||
if not destination.exists():
|
||||
if backup.exists():
|
||||
# If restoration fails, do not discard either recoverable directory.
|
||||
_move_directory(backup, destination, parent)
|
||||
elif candidate.exists() and complete(candidate):
|
||||
_move_directory(candidate, destination, parent)
|
||||
elif state["had_destination"]:
|
||||
raise RuntimeError("Original staging directory and backup are both missing; recovery retained")
|
||||
if destination.exists() and backup.exists():
|
||||
if not complete(destination):
|
||||
raise RuntimeError("Ambiguous resource staging recovery; original backup retained")
|
||||
_remove_tree(backup, parent)
|
||||
_remove_tree(candidate, parent)
|
||||
if destination.exists():
|
||||
completion = destination / marker
|
||||
if completion.exists():
|
||||
if not complete(destination):
|
||||
raise RuntimeError("Invalid resource staging completion marker")
|
||||
completion.unlink()
|
||||
journal.unlink()
|
||||
|
||||
|
||||
def _previous_owned(destination):
|
||||
names = {"searchi-gemma-worker.exe", "searchi-gemma-worker", "runtime-libraries.json",
|
||||
"LLAMA-LICENSE.txt", "VULKAN-LICENSE.txt", "MSVC-REDISTRIBUTION.txt", "vulkan-1.dll"}
|
||||
manifest_path = Path(__file__).with_name("model-manifest.json")
|
||||
if manifest_path.stat().st_size > 65536:
|
||||
raise RuntimeError("Pinned model manifest is oversized")
|
||||
model_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
files = model_manifest.get("files") if isinstance(model_manifest, dict) else None
|
||||
if not isinstance(files, list) or not 1 <= len(files) <= 4:
|
||||
raise RuntimeError("Invalid pinned model manifest")
|
||||
for record in files:
|
||||
name = record.get("path") if isinstance(record, dict) else None
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.gguf", name):
|
||||
raise RuntimeError("Invalid pinned model resource path")
|
||||
names.add(f"embeddinggemma2/{name}")
|
||||
inventory = destination / "runtime-libraries.json"
|
||||
if inventory.exists():
|
||||
if inventory.stat().st_size > 65536:
|
||||
raise RuntimeError("Previous runtime inventory is oversized")
|
||||
previous = json.loads(inventory.read_text(encoding="utf-8"))
|
||||
records = previous.get("libraries") if isinstance(previous, dict) else None
|
||||
if not isinstance(records, list) or len(records) > 64:
|
||||
raise RuntimeError("Invalid previous runtime inventory")
|
||||
for record in records:
|
||||
name = record.get("path") if isinstance(record, dict) else None
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+\.dll", name):
|
||||
raise RuntimeError("Invalid previous runtime library path")
|
||||
names.add(name)
|
||||
return names
|
||||
|
||||
|
||||
def _resource_key(path):
|
||||
value = Path(path).as_posix()
|
||||
return value.casefold() if os.name == "nt" else value
|
||||
|
||||
|
||||
def _preserved_snapshot(destination, owned):
|
||||
"""Bounded content/tree snapshot detects edits to resources being retained."""
|
||||
if not destination.exists():
|
||||
return None
|
||||
_check_tree(destination)
|
||||
snapshot = {}
|
||||
count = 0
|
||||
for parent, directories, files in os.walk(destination, followlinks=False):
|
||||
for name in [*directories, *files]:
|
||||
count += 1
|
||||
if count > 100000:
|
||||
raise RuntimeError("Resource staging exceeds its snapshot file-count limit")
|
||||
path = Path(parent) / name
|
||||
key = _resource_key(path.relative_to(destination))
|
||||
if key in owned:
|
||||
if not path.is_file():
|
||||
raise RuntimeError("Owned runtime resource is not a file")
|
||||
continue
|
||||
if path.is_dir():
|
||||
snapshot[key] = ("directory",)
|
||||
else:
|
||||
record = _record(path)
|
||||
metadata = path.stat()
|
||||
snapshot[key] = ("file", *record, metadata.st_mtime_ns, metadata.st_ino, metadata.st_dev)
|
||||
return snapshot
|
||||
|
||||
|
||||
def stage_resources(destination, sources, validate=None):
|
||||
"""Replace a verified complete resource directory and preserve other assets."""
|
||||
destination = Path(destination).absolute()
|
||||
if destination.parent != destination.parent.resolve():
|
||||
raise RuntimeError("Staging parent resolves through a link outside its expected location")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
parent = destination.parent.resolve()
|
||||
destination = _checked_path(destination, parent)
|
||||
with staging_lock(parent):
|
||||
recover_staging(destination)
|
||||
if destination.exists():
|
||||
_check_tree(destination)
|
||||
if (destination / ".searchi-stage-complete").exists():
|
||||
raise RuntimeError("Reserved staging completion marker exists without its journal")
|
||||
owned = _previous_owned(destination)
|
||||
owned_keys = {_resource_key(path) for path in owned}
|
||||
preserved = _preserved_snapshot(destination, owned_keys)
|
||||
records = []
|
||||
targets = set()
|
||||
for source, relative in sources:
|
||||
relative = Path(relative)
|
||||
if relative.is_absolute() or relative.anchor or relative.drive or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("Invalid staging resource target")
|
||||
key = relative.as_posix().lower()
|
||||
if key in targets:
|
||||
raise RuntimeError("Duplicate staging resource target")
|
||||
targets.add(key)
|
||||
records.append((Path(source), relative, _record(source)))
|
||||
state = {"schema": 1, "owner": "searchi-runtime-staging", "transaction": uuid.uuid4().hex,
|
||||
"had_destination": destination.exists()}
|
||||
candidate, backup = _journal_paths(destination, state)
|
||||
journal = _checked_path(parent / ".searchi-stage.json", parent)
|
||||
pending = _checked_path(parent / f".searchi-stage-journal-{state['transaction']}.pending", parent)
|
||||
try:
|
||||
with pending.open("x", encoding="utf-8") as stream:
|
||||
json.dump(state, stream)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(pending, journal)
|
||||
finally:
|
||||
pending.unlink(missing_ok=True)
|
||||
try:
|
||||
if destination.exists():
|
||||
def ignore_owned(directory, names):
|
||||
relative = Path(directory).relative_to(destination)
|
||||
return [name for name in names if _resource_key(relative / name) in owned_keys]
|
||||
|
||||
shutil.copytree(destination, candidate, copy_function=_verified_copy, ignore=ignore_owned)
|
||||
else:
|
||||
candidate.mkdir()
|
||||
model_directory = candidate / "embeddinggemma2"
|
||||
if model_directory.is_dir() and not any(model_directory.iterdir()):
|
||||
if not model_directory.resolve().is_relative_to(candidate.resolve()):
|
||||
raise RuntimeError("Owned model directory escapes candidate")
|
||||
model_directory.rmdir()
|
||||
for source, relative, expected in records:
|
||||
target = candidate / relative
|
||||
if not target.resolve().is_relative_to(candidate.resolve()):
|
||||
raise RuntimeError("Staging resource target escapes candidate directory")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
_verified_copy(source, target, expected)
|
||||
_check_tree(candidate)
|
||||
if validate:
|
||||
validate(candidate)
|
||||
if _preserved_snapshot(destination, owned_keys) != preserved:
|
||||
raise RuntimeError("Preserved resources changed during staging; original retained")
|
||||
(candidate / ".searchi-stage-complete").write_text(state["transaction"])
|
||||
if destination.exists():
|
||||
_move_directory(destination, backup, parent)
|
||||
_move_directory(candidate, destination, parent)
|
||||
recover_staging(destination)
|
||||
except Exception:
|
||||
recover_staging(destination)
|
||||
raise
|
||||
|
||||
|
||||
def verify_runtime(source, required=False):
|
||||
"""Preflight an explicit, bounded dependency inventory, never a DLL glob."""
|
||||
inventory = source.parent / "runtime-libraries.json"
|
||||
@@ -74,6 +385,12 @@ def main():
|
||||
parser.add_argument("--backend", choices=["cpu", "auto", "metal", "vulkan", "cuda"], default="auto")
|
||||
parser.add_argument("--bundle-model", action="store_true")
|
||||
args = parser.parse_args()
|
||||
destination = ROOT / "apps/desktop/src-tauri/resources"
|
||||
if destination.parent.absolute() != destination.parent.resolve():
|
||||
raise RuntimeError("Staging parent resolves through a link outside its expected location")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with staging_lock(destination.parent):
|
||||
recover_staging(destination)
|
||||
name = "searchi-gemma-worker.exe" if os.name == "nt" else "searchi-gemma-worker"
|
||||
source = ROOT / ".searchi/runtime" / args.backend
|
||||
if os.name == "nt":
|
||||
@@ -91,29 +408,30 @@ def main():
|
||||
raise RuntimeError("Microsoft CRT redistribution notice is missing")
|
||||
dependencies.append(crt_notice)
|
||||
artifacts = []
|
||||
model_manifest = None
|
||||
if args.bundle_model:
|
||||
manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||
artifacts = verify_bundle(ROOT, manifest)
|
||||
model_manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||
artifacts = verify_bundle(ROOT, model_manifest)
|
||||
loader = None
|
||||
if os.name == "nt" and args.backend in {"auto", "vulkan"}:
|
||||
from provision_vulkan import provision
|
||||
_, loader = provision()
|
||||
destination = ROOT / "apps/desktop/src-tauri/resources"
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, destination / name)
|
||||
for dependency in dependencies:
|
||||
shutil.copy2(dependency, destination / dependency.name)
|
||||
# Include the upstream runtime license with the statically linked helper.
|
||||
shutil.copy2(license_path, destination / "LLAMA-LICENSE.txt")
|
||||
sources = [(source, source.name), (license_path, "LLAMA-LICENSE.txt")]
|
||||
sources.extend((path, path.name) for path in dependencies)
|
||||
if loader:
|
||||
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
|
||||
if name not in {path.name for path in dependencies}:
|
||||
shutil.copy2(loader / name, destination / name)
|
||||
if args.bundle_model:
|
||||
model_destination = destination / "embeddinggemma2"
|
||||
model_destination.mkdir(exist_ok=True)
|
||||
for artifact in artifacts:
|
||||
shutil.copy2(artifact, model_destination / artifact.name)
|
||||
sources.append((loader / name, name))
|
||||
sources.extend((path, f"embeddinggemma2/{path.name}") for path in artifacts)
|
||||
|
||||
def validate(candidate):
|
||||
verify_runtime(candidate / source.name, os.name == "nt" and args.backend in {"auto", "vulkan"})
|
||||
if model_manifest:
|
||||
for artifact, spec in zip(artifacts, model_manifest["files"], strict=True):
|
||||
if _record(candidate / "embeddinggemma2" / artifact.name) != (spec["bytes"], spec["sha256"]):
|
||||
raise RuntimeError("Copied model checksum mismatch")
|
||||
|
||||
stage_resources(destination, sources, validate)
|
||||
print(f"Staged native worker: {destination / source.name}")
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -11,6 +12,371 @@ import stage_runtime
|
||||
|
||||
|
||||
class StagingTests(unittest.TestCase):
|
||||
def resources(self, directory):
|
||||
root = Path(directory)
|
||||
destination = root / "src-tauri/resources"
|
||||
destination.mkdir(parents=True)
|
||||
(destination / "searchi-gemma-worker.exe").write_bytes(b"previous worker")
|
||||
(destination / "fonts").mkdir()
|
||||
(destination / "fonts/Inter.woff2").write_bytes(b"font")
|
||||
(destination / "media-runtime").mkdir()
|
||||
(destination / "media-runtime/ffmpeg.exe").write_bytes(b"media")
|
||||
(destination / "user-note.txt").write_bytes(b"preserve user file")
|
||||
source = root / "worker.exe"
|
||||
source.write_bytes(b"next worker")
|
||||
return destination, source
|
||||
|
||||
def test_failed_copy_preserves_previous_complete_resources(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.shutil.copy2
|
||||
|
||||
def fail_worker(path, target, *args, **kwargs):
|
||||
if Path(path) == source:
|
||||
raise OSError("injected copy failure")
|
||||
return original(path, target, *args, **kwargs)
|
||||
|
||||
with patch.object(stage_runtime.shutil, "copy2", fail_worker):
|
||||
with self.assertRaisesRegex(OSError, "copy failure"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
self.assertEqual((destination / "user-note.txt").read_bytes(), b"preserve user file")
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
self.assertEqual(list(destination.parent.glob(".resources.stage-*")), [])
|
||||
|
||||
def test_changed_copy_bytes_reject_before_swap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.shutil.copy2
|
||||
|
||||
def corrupt_worker(path, target, *args, **kwargs):
|
||||
result = original(path, target, *args, **kwargs)
|
||||
if Path(path) == source:
|
||||
Path(target).write_bytes(b"bad worker!")
|
||||
return result
|
||||
|
||||
with patch.object(stage_runtime.shutil, "copy2", corrupt_worker):
|
||||
with self.assertRaisesRegex(RuntimeError, "changed or copy was incomplete"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
|
||||
def test_source_change_after_preflight_rejects_before_swap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.shutil.copy2
|
||||
|
||||
def mutate_source(path, target, *args, **kwargs):
|
||||
if Path(path) == source:
|
||||
source.write_bytes(b"evil worker")
|
||||
return original(path, target, *args, **kwargs)
|
||||
|
||||
with patch.object(stage_runtime.shutil, "copy2", mutate_source):
|
||||
with self.assertRaisesRegex(RuntimeError, "changed or copy was incomplete"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
|
||||
def test_resource_targets_reject_traversal_and_windows_drive_roots(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
targets = ["../outside"]
|
||||
if os.name == "nt":
|
||||
targets.extend(["C:outside", "\\outside"])
|
||||
for target in targets:
|
||||
with self.subTest(target=target), self.assertRaisesRegex(RuntimeError, "resource target"):
|
||||
stage_runtime.stage_resources(destination, [(source, target)])
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
|
||||
def test_locked_destination_rename_preserves_old_resources(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.os.replace
|
||||
|
||||
def fail_old_rename(path, target):
|
||||
if Path(path) == destination:
|
||||
raise PermissionError("destination is locked")
|
||||
return original(path, target)
|
||||
|
||||
with patch.object(stage_runtime.os, "replace", fail_old_rename):
|
||||
with self.assertRaisesRegex(PermissionError, "locked"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
|
||||
def test_interrupted_between_renames_restores_old_before_source_preflight(self):
|
||||
class Interrupt(BaseException):
|
||||
pass
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.os.replace
|
||||
|
||||
def interrupt_candidate(path, target):
|
||||
if Path(path).name.startswith(".resources.stage-"):
|
||||
raise Interrupt()
|
||||
return original(path, target)
|
||||
|
||||
with patch.object(stage_runtime.os, "replace", interrupt_candidate):
|
||||
with self.assertRaises(Interrupt):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertFalse(destination.exists())
|
||||
self.assertEqual(len(list(destination.parent.glob(".resources.backup-*"))), 1)
|
||||
with self.assertRaisesRegex(RuntimeError, "source is missing"):
|
||||
stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
|
||||
def test_interrupted_after_install_commits_verified_new_resources(self):
|
||||
class Interrupt(BaseException):
|
||||
pass
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.os.replace
|
||||
|
||||
def interrupt_after_candidate(path, target):
|
||||
original(path, target)
|
||||
if Path(path).name.startswith(".resources.stage-"):
|
||||
raise Interrupt()
|
||||
|
||||
with patch.object(stage_runtime.os, "replace", interrupt_after_candidate):
|
||||
with self.assertRaises(Interrupt):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
with self.assertRaisesRegex(RuntimeError, "source is missing"):
|
||||
stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"next worker")
|
||||
self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
|
||||
self.assertEqual((destination / "media-runtime/ffmpeg.exe").read_bytes(), b"media")
|
||||
self.assertEqual((destination / "user-note.txt").read_bytes(), b"preserve user file")
|
||||
self.assertEqual(list(destination.parent.glob(".resources.backup-*")), [])
|
||||
self.assertFalse((destination / ".searchi-stage-complete").exists())
|
||||
|
||||
def test_failed_rollback_retains_backup_and_journal_for_retry(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.os.replace
|
||||
|
||||
def fail_candidate_and_restore(path, target):
|
||||
if Path(path).name.startswith((".resources.stage-", ".resources.backup-")):
|
||||
raise PermissionError("injected rename failure")
|
||||
return original(path, target)
|
||||
|
||||
with patch.object(stage_runtime.os, "replace", fail_candidate_and_restore):
|
||||
with self.assertRaisesRegex(PermissionError, "rename failure"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
backup = next(destination.parent.glob(".resources.backup-*"))
|
||||
self.assertEqual((backup / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
self.assertTrue((destination.parent / ".searchi-stage.json").exists())
|
||||
with self.assertRaisesRegex(RuntimeError, "source is missing"):
|
||||
stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
|
||||
def test_profile_switch_removes_only_previous_inventory_libraries(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
old = destination / "ggml-cpu-old.dll"
|
||||
old.write_bytes(b"old backend")
|
||||
(destination / "user-plugin.dll").write_bytes(b"user plugin")
|
||||
(destination / "runtime-libraries.json").write_text(json.dumps({"libraries": [{"path": old.name}]}))
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertFalse((destination / old.name).exists())
|
||||
self.assertFalse((destination / "runtime-libraries.json").exists())
|
||||
self.assertEqual((destination / "user-plugin.dll").read_bytes(), b"user plugin")
|
||||
self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
|
||||
|
||||
def test_offline_to_online_removes_only_exact_pinned_model_pair(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
model_directory = destination / "embeddinggemma2"
|
||||
model_directory.mkdir()
|
||||
manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
|
||||
for spec in manifest["files"]:
|
||||
(model_directory / spec["path"]).write_bytes(b"old unverified model")
|
||||
(model_directory / "user-note.txt").write_bytes(b"preserve unrelated asset")
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
for spec in manifest["files"]:
|
||||
self.assertFalse((destination / "embeddinggemma2" / spec["path"]).exists())
|
||||
self.assertEqual((destination / "embeddinggemma2/user-note.txt").read_bytes(), b"preserve unrelated asset")
|
||||
self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
|
||||
|
||||
def test_online_staging_removes_empty_owned_model_directory(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
model_directory = destination / "embeddinggemma2"
|
||||
model_directory.mkdir()
|
||||
manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
|
||||
for spec in manifest["files"]:
|
||||
(model_directory / spec["path"]).write_bytes(b"old model")
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertFalse((destination / "embeddinggemma2").exists())
|
||||
|
||||
def test_old_owned_worker_and_model_pair_are_never_copied(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
model_directory = destination / "embeddinggemma2"
|
||||
model_directory.mkdir()
|
||||
manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
|
||||
old_paths = {destination / "searchi-gemma-worker.exe"}
|
||||
for spec in manifest["files"]:
|
||||
path = model_directory / spec["path"]
|
||||
path.write_bytes(b"old owned model")
|
||||
old_paths.add(path)
|
||||
original = stage_runtime.shutil.copy2
|
||||
|
||||
original_record = stage_runtime._record
|
||||
|
||||
def reject_owned_copy(path, target, *args, **kwargs):
|
||||
if Path(path) in old_paths:
|
||||
self.fail("Previously owned runtime/model asset was unnecessarily copied")
|
||||
return original(path, target, *args, **kwargs)
|
||||
|
||||
def reject_owned_hash(path):
|
||||
if Path(path) in old_paths:
|
||||
self.fail("Previously owned runtime/model asset was unnecessarily hashed")
|
||||
return original_record(path)
|
||||
|
||||
with patch.object(stage_runtime.shutil, "copy2", reject_owned_copy), patch.object(stage_runtime, "_record", reject_owned_hash):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"next worker")
|
||||
self.assertFalse((destination / "embeddinggemma2").exists())
|
||||
|
||||
def test_concurrent_preserved_edit_aborts_and_keeps_latest_original(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
|
||||
def edit_original(_candidate):
|
||||
(destination / "user-note.txt").write_bytes(b"latest user edit")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "Preserved resources changed"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")], edit_original)
|
||||
self.assertEqual((destination / "user-note.txt").read_bytes(), b"latest user edit")
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
|
||||
def test_concurrent_preserved_addition_aborts_and_keeps_new_file(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
|
||||
def add_original(_candidate):
|
||||
(destination / "fonts/new-font.woff2").write_bytes(b"new user asset")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "Preserved resources changed"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")], add_original)
|
||||
self.assertEqual((destination / "fonts/new-font.woff2").read_bytes(), b"new user asset")
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
|
||||
def test_concurrent_staging_lock_fails_without_mutation(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
with stage_runtime.staging_lock(destination.parent):
|
||||
with self.assertRaisesRegex(RuntimeError, "holds the lock"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
|
||||
def test_malicious_recovery_journal_cannot_escape_sibling_paths(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
outside = Path(directory) / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "user-data.txt").write_bytes(b"never remove")
|
||||
journal = destination.parent / ".searchi-stage.json"
|
||||
journal.write_text(json.dumps({"schema": 1, "owner": "searchi-runtime-staging", "transaction": "../outside", "had_destination": True}))
|
||||
with self.assertRaisesRegex(RuntimeError, "recovery journal"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((outside / "user-data.txt").read_bytes(), b"never remove")
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
|
||||
def test_existing_unrelated_symlink_rejects_before_stage_changes(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
outside = Path(directory) / "outside.txt"
|
||||
outside.write_bytes(b"private data")
|
||||
link = destination / "user-link.txt"
|
||||
try:
|
||||
link.symlink_to(outside)
|
||||
except OSError:
|
||||
self.skipTest("Host does not permit creating test symlinks")
|
||||
try:
|
||||
with self.assertRaisesRegex(RuntimeError, "symlinks or junctions"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual(outside.read_bytes(), b"private data")
|
||||
self.assertTrue(link.is_symlink())
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
finally:
|
||||
link.unlink()
|
||||
|
||||
def test_first_install_interrupt_after_verified_candidate_recovers(self):
|
||||
class Interrupt(BaseException):
|
||||
pass
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
destination = root / "src-tauri/resources"
|
||||
source = root / "worker.exe"
|
||||
source.write_bytes(b"first worker")
|
||||
original = stage_runtime.os.replace
|
||||
|
||||
def interrupt_candidate(path, target):
|
||||
if Path(path).name.startswith(".resources.stage-"):
|
||||
raise Interrupt()
|
||||
return original(path, target)
|
||||
|
||||
with patch.object(stage_runtime.os, "replace", interrupt_candidate):
|
||||
with self.assertRaises(Interrupt):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertFalse(destination.exists())
|
||||
with stage_runtime.staging_lock(destination.parent):
|
||||
stage_runtime.recover_staging(destination)
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"first worker")
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
|
||||
@unittest.skipUnless(os.name == "nt", "Windows junction boundary regression")
|
||||
def test_existing_junction_rejects_without_following_outside_directory(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
outside = Path(directory) / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "private.txt").write_bytes(b"untouched")
|
||||
link = destination / "user-junction"
|
||||
result = subprocess.run(["cmd.exe", "/c", "mklink", "/J", str(link), str(outside)], capture_output=True)
|
||||
if result.returncode:
|
||||
self.skipTest("Host does not permit creating a test junction")
|
||||
try:
|
||||
self.assertTrue(stage_runtime._is_link(link))
|
||||
with self.assertRaisesRegex(RuntimeError, "symlinks or junctions"):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((outside / "private.txt").read_bytes(), b"untouched")
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
finally:
|
||||
# Remove only the verified test junction, never its target tree.
|
||||
self.assertEqual(link.absolute().parent, destination.absolute())
|
||||
self.assertTrue(stage_runtime._is_link(link))
|
||||
link.rmdir()
|
||||
|
||||
def test_interrupted_partial_candidate_keeps_old_resources_and_recovers(self):
|
||||
class Interrupt(BaseException):
|
||||
pass
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
destination, source = self.resources(directory)
|
||||
original = stage_runtime.shutil.copy2
|
||||
|
||||
def interrupt_copy(path, target, *args, **kwargs):
|
||||
if Path(path) == source:
|
||||
Path(target).write_bytes(b"partial")
|
||||
raise Interrupt()
|
||||
return original(path, target, *args, **kwargs)
|
||||
|
||||
with patch.object(stage_runtime.shutil, "copy2", interrupt_copy):
|
||||
with self.assertRaises(Interrupt):
|
||||
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||
with stage_runtime.staging_lock(destination.parent):
|
||||
stage_runtime.recover_staging(destination)
|
||||
self.assertEqual(list(destination.parent.glob(".resources.stage-*")), [])
|
||||
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||
|
||||
@unittest.skipUnless(os.name == "nt", "Windows app-local CRT staging")
|
||||
def test_missing_crt_notice_preserves_existing_staged_worker(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
Reference in new issue
Block a user