Harden worker IPC and recoverable runtime staging
This commit is contained in:
1 parent
0bba1a0aa7
commit
b3ebd7a894
12 files changed
+1146
-69
No files matched your search
@@ -7,6 +7,10 @@ dist/
|
|||||||
.searchi/
|
.searchi/
|
||||||
__pycache__/
|
__pycache__/
|
||||||
/apps/desktop/src-tauri/gen/
|
/apps/desktop/src-tauri/gen/
|
||||||
|
/apps/desktop/src-tauri/.searchi-stage.lock
|
||||||
|
/apps/desktop/src-tauri/.searchi-stage.json
|
||||||
|
/apps/desktop/src-tauri/.resources.stage-*/
|
||||||
|
/apps/desktop/src-tauri/.resources.backup-*/
|
||||||
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
|
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
|
||||||
/apps/desktop/src-tauri/resources/vulkan-1.dll
|
/apps/desktop/src-tauri/resources/vulkan-1.dll
|
||||||
/apps/desktop/src-tauri/resources/*.dll
|
/apps/desktop/src-tauri/resources/*.dll
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
//! Explicit data-directory isolation for portable use and native release QA.
|
||||||
|
//! Ordinary launches continue to use the operating system's app-data directory.
|
||||||
|
use std::{ffi::OsString, io, path::PathBuf};
|
||||||
|
|
||||||
|
pub fn resolve(default: PathBuf, override_path: Option<OsString>) -> io::Result<PathBuf> {
|
||||||
|
let Some(value) = override_path else {
|
||||||
|
return Ok(default);
|
||||||
|
};
|
||||||
|
let path = PathBuf::from(value);
|
||||||
|
if !path.is_absolute() || path.as_os_str().len() > 32768 {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidInput,
|
||||||
|
"SEARCHI_DATA_DIR must be an absolute bounded directory path",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_directory_isolates_settings_and_index_without_changing_default() {
|
||||||
|
let default = std::env::temp_dir().join("searchi-default");
|
||||||
|
let isolated = std::env::temp_dir().join("searchi-isolated");
|
||||||
|
assert_eq!(resolve(default.clone(), None).unwrap(), default);
|
||||||
|
assert_eq!(
|
||||||
|
resolve(default, Some(isolated.clone().into_os_string())).unwrap(),
|
||||||
|
isolated
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn invalid_override_is_rejected_without_falling_back_to_user_data() {
|
||||||
|
let default = std::env::temp_dir().join("searchi-default");
|
||||||
|
for value in [OsString::new(), OsString::from("relative/data")] {
|
||||||
|
assert_eq!(
|
||||||
|
resolve(default.clone(), Some(value)).unwrap_err().kind(),
|
||||||
|
io::ErrorKind::InvalidInput
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,10 @@
|
|||||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||||
|
|
||||||
|
#[cfg(all(not(debug_assertions), not(feature = "custom-protocol")))]
|
||||||
|
compile_error!("Release desktop builds require --features custom-protocol to embed the frontend");
|
||||||
|
|
||||||
mod ai_commands;
|
mod ai_commands;
|
||||||
|
mod data_directory;
|
||||||
mod model_download;
|
mod model_download;
|
||||||
mod semantic_commands;
|
mod semantic_commands;
|
||||||
mod settings;
|
mod settings;
|
||||||
@@ -394,7 +398,10 @@ fn main() {
|
|||||||
}
|
}
|
||||||
}).build())
|
}).build())
|
||||||
.setup(|app| {
|
.setup(|app| {
|
||||||
let directory = app.path().app_data_dir()?;
|
let directory = data_directory::resolve(
|
||||||
|
app.path().app_data_dir()?,
|
||||||
|
std::env::var_os("SEARCHI_DATA_DIR"),
|
||||||
|
)?;
|
||||||
std::fs::create_dir_all(&directory)?;
|
std::fs::create_dir_all(&directory)?;
|
||||||
let database = directory.join("searchi.sqlite");
|
let database = directory.join("searchi.sqlite");
|
||||||
let lexical = Arc::new(Mutex::new(SearchIndex::open(&database)?));
|
let lexical = Arc::new(Mutex::new(SearchIndex::open(&database)?));
|
||||||
@@ -413,15 +420,15 @@ fn main() {
|
|||||||
let development_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..");
|
let development_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..");
|
||||||
let development_models = development_root.join(".searchi/models/embeddinggemma2");
|
let development_models = development_root.join(".searchi/models/embeddinggemma2");
|
||||||
let bundled_models = app.path().resource_dir()?.join("embeddinggemma2");
|
let bundled_models = app.path().resource_dir()?.join("embeddinggemma2");
|
||||||
let model_directory = settings.value.model_directory.clone().unwrap_or_else(|| {
|
let model_directory = if let Some(configured) = settings.value.model_directory.clone() {
|
||||||
if cfg!(debug_assertions) && development_models.is_dir() {
|
configured
|
||||||
|
} else if cfg!(debug_assertions) && development_models.is_dir() {
|
||||||
development_models.canonicalize().unwrap_or(development_models)
|
development_models.canonicalize().unwrap_or(development_models)
|
||||||
} else if bundled_models.is_dir() {
|
} else if searchi_assets::has_model_files(&bundled_models)? {
|
||||||
bundled_models
|
bundled_models
|
||||||
} else {
|
} else {
|
||||||
searchi_assets::default_model_directory().unwrap_or_else(|_| directory.join("Models"))
|
searchi_assets::default_model_directory().unwrap_or_else(|_| directory.join("Models"))
|
||||||
}
|
};
|
||||||
});
|
|
||||||
let worker_name = if cfg!(windows) { "searchi-gemma-worker.exe" } else { "searchi-gemma-worker" };
|
let worker_name = if cfg!(windows) { "searchi-gemma-worker.exe" } else { "searchi-gemma-worker" };
|
||||||
let development_worker = development_root.join(".searchi/runtime/cpu")
|
let development_worker = development_root.join(".searchi/runtime/cpu")
|
||||||
.join(if cfg!(windows) { "Release" } else { "" }).join(worker_name);
|
.join(if cfg!(windows) { "Release" } else { "" }).join(worker_name);
|
||||||
|
|||||||
@@ -94,6 +94,27 @@ pub fn default_model_directory() -> Result<PathBuf> {
|
|||||||
.join("embeddinggemma2"))
|
.join("embeddinggemma2"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cheap location selection for optional bundled assets. This checks regular
|
||||||
|
/// files and expected lengths only; `verify_cached` still verifies SHA-256
|
||||||
|
/// before setup considers the model usable. Partial bundles use managed cache.
|
||||||
|
pub fn has_model_files(directory: &Path) -> Result<bool> {
|
||||||
|
model_files_present(directory, &manifest()?.files)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn model_files_present(directory: &Path, artifacts: &[Artifact]) -> Result<bool> {
|
||||||
|
for artifact in artifacts {
|
||||||
|
let metadata = match fs::metadata(directory.join(&artifact.path)) {
|
||||||
|
Ok(metadata) => metadata,
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
|
||||||
|
Err(error) => return Err(error).context("inspect bundled model artifact"),
|
||||||
|
};
|
||||||
|
if !metadata.is_file() || metadata.len() != artifact.bytes {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
/// Offline validation includes file length and full SHA-256, not mere existence.
|
/// Offline validation includes file length and full SHA-256, not mere existence.
|
||||||
pub fn verify_cached(directory: &Path) -> Result<bool> {
|
pub fn verify_cached(directory: &Path) -> Result<bool> {
|
||||||
for artifact in manifest()?.files {
|
for artifact in manifest()?.files {
|
||||||
@@ -383,6 +404,32 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn optional_bundle_requires_both_regular_files_at_expected_sizes() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let artifacts = [artifact("one.gguf", b"abc"), artifact("two.gguf", b"xyz")];
|
||||||
|
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||||
|
fs::write(directory.path().join("one.gguf"), b"abc").unwrap();
|
||||||
|
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||||
|
fs::create_dir(directory.path().join("two.gguf")).unwrap();
|
||||||
|
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||||
|
fs::remove_dir(directory.path().join("two.gguf")).unwrap();
|
||||||
|
fs::write(directory.path().join("two.gguf"), b"xyz").unwrap();
|
||||||
|
assert!(model_files_present(directory.path(), &artifacts).unwrap());
|
||||||
|
fs::write(directory.path().join("one.gguf"), b"ab").unwrap();
|
||||||
|
assert!(!model_files_present(directory.path(), &artifacts).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bundled_file_presence_never_substitutes_for_sha_verification() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let artifact = artifact("one.gguf", b"abc");
|
||||||
|
let path = directory.path().join(&artifact.path);
|
||||||
|
fs::write(&path, b"xyz").unwrap();
|
||||||
|
assert!(model_files_present(directory.path(), std::slice::from_ref(&artifact)).unwrap());
|
||||||
|
assert!(!verified(&path, &artifact).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn same_size_corruption_is_detected_and_missing_cache_is_offline() {
|
fn same_size_corruption_is_detected_and_missing_cache_is_offline() {
|
||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -60,7 +60,10 @@ fn query_operation(info: &WorkerInfo, code: bool) -> Result<&'static str> {
|
|||||||
/// Persistent isolated native worker. No HTTP or remote model fallback.
|
/// Persistent isolated native worker. No HTTP or remote model fallback.
|
||||||
pub struct GemmaEmbedder {
|
pub struct GemmaEmbedder {
|
||||||
child: Child,
|
child: Child,
|
||||||
stdin: ChildStdin,
|
requests: Option<mpsc::SyncSender<Vec<u8>>>,
|
||||||
|
writes: Receiver<std::result::Result<(), String>>,
|
||||||
|
writer_thread: Option<thread::JoinHandle<()>>,
|
||||||
|
reader_thread: Option<thread::JoinHandle<()>>,
|
||||||
responses: Receiver<std::result::Result<Value, String>>,
|
responses: Receiver<std::result::Result<Value, String>>,
|
||||||
pub info: WorkerInfo,
|
pub info: WorkerInfo,
|
||||||
healthy: bool,
|
healthy: bool,
|
||||||
@@ -73,24 +76,48 @@ pub struct GemmaEmbedder {
|
|||||||
|
|
||||||
impl Drop for GemmaEmbedder {
|
impl Drop for GemmaEmbedder {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
let _ = self.child.kill();
|
self.stop_worker();
|
||||||
let _ = self.child.wait();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type WriterTransport = (
|
||||||
|
mpsc::SyncSender<Vec<u8>>,
|
||||||
|
Receiver<std::result::Result<(), String>>,
|
||||||
|
thread::JoinHandle<()>,
|
||||||
|
);
|
||||||
|
|
||||||
|
fn spawn_writer(mut stdin: ChildStdin) -> WriterTransport {
|
||||||
|
let (requests, pending) = mpsc::sync_channel::<Vec<u8>>(1);
|
||||||
|
let (completed, writes) = mpsc::sync_channel(1);
|
||||||
|
let writer = thread::spawn(move || {
|
||||||
|
while let Ok(encoded) = pending.recv() {
|
||||||
|
let result = stdin
|
||||||
|
.write_all(&encoded)
|
||||||
|
.and_then(|_| stdin.write_all(b"\n"))
|
||||||
|
.and_then(|_| stdin.flush())
|
||||||
|
.map_err(|error| format!("Cannot write local Gemma worker request: {error}"));
|
||||||
|
let failed = result.is_err();
|
||||||
|
if completed.try_send(result).is_err() || failed {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
(requests, writes, writer)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct StartupTransportFailure(String);
|
struct WorkerTransportFailure(String);
|
||||||
impl std::fmt::Display for StartupTransportFailure {
|
impl std::fmt::Display for WorkerTransportFailure {
|
||||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
formatter.write_str(&self.0)
|
formatter.write_str(&self.0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
impl std::error::Error for StartupTransportFailure {}
|
impl std::error::Error for WorkerTransportFailure {}
|
||||||
|
|
||||||
/// Retry only process/transport failure, never an incompatible protocol or model.
|
/// Retry only process/transport failure, never an incompatible protocol or model.
|
||||||
fn start_with_cpu_fallback<T>(cpu: bool, mut start: impl FnMut(bool) -> Result<T>) -> Result<T> {
|
fn start_with_cpu_fallback<T>(cpu: bool, mut start: impl FnMut(bool) -> Result<T>) -> Result<T> {
|
||||||
match start(cpu) {
|
match start(cpu) {
|
||||||
Err(error) if !cpu && error.downcast_ref::<StartupTransportFailure>().is_some() => {
|
Err(error) if !cpu && error.downcast_ref::<WorkerTransportFailure>().is_some() => {
|
||||||
start(true).with_context(|| {
|
start(true).with_context(|| {
|
||||||
format!("CPU initialization failed after automatic worker startup failure: {error}")
|
format!("CPU initialization failed after automatic worker startup failure: {error}")
|
||||||
})
|
})
|
||||||
@@ -163,7 +190,7 @@ impl GemmaEmbedder {
|
|||||||
command.creation_flags(0x08000000); // CREATE_NO_WINDOW
|
command.creation_flags(0x08000000); // CREATE_NO_WINDOW
|
||||||
}
|
}
|
||||||
let mut child = command.spawn().map_err(|error| {
|
let mut child = command.spawn().map_err(|error| {
|
||||||
StartupTransportFailure(format!("Cannot start local Gemma worker: {error}"))
|
WorkerTransportFailure(format!("Cannot start local Gemma worker: {error}"))
|
||||||
})?;
|
})?;
|
||||||
let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else {
|
let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else {
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
@@ -171,12 +198,12 @@ impl GemmaEmbedder {
|
|||||||
anyhow::bail!("Gemma worker protocol pipes missing");
|
anyhow::bail!("Gemma worker protocol pipes missing");
|
||||||
};
|
};
|
||||||
let (sender, responses) = mpsc::sync_channel(1);
|
let (sender, responses) = mpsc::sync_channel(1);
|
||||||
thread::spawn(move || {
|
let reader_thread = thread::spawn(move || {
|
||||||
let mut reader = BufReader::new(stdout);
|
let mut reader = BufReader::new(stdout);
|
||||||
loop {
|
loop {
|
||||||
let response = read_response(&mut reader);
|
let response = read_response(&mut reader);
|
||||||
let failed = response.is_err();
|
let failed = response.is_err();
|
||||||
if sender.send(response).is_err() || failed {
|
if sender.try_send(response).is_err() || failed {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -186,13 +213,14 @@ impl GemmaEmbedder {
|
|||||||
other => {
|
other => {
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
let _ = child.wait();
|
let _ = child.wait();
|
||||||
|
let _ = reader_thread.join();
|
||||||
// Malformed/oversized JSON is a protocol violation, not evidence
|
// Malformed/oversized JSON is a protocol violation, not evidence
|
||||||
// that hardware acceleration failed. Do not mask it with retry.
|
// that hardware acceleration failed. Do not mask it with retry.
|
||||||
if matches!(&other, Ok(Err(error)) if error.starts_with("Invalid Gemma worker JSON:") || error == "Gemma worker response exceeds protocol byte budget")
|
if matches!(&other, Ok(Err(error)) if error.starts_with("Invalid Gemma worker JSON:") || error == "Gemma worker response exceeds protocol byte budget")
|
||||||
{
|
{
|
||||||
anyhow::bail!("Invalid Gemma worker startup protocol: {other:?}");
|
anyhow::bail!("Invalid Gemma worker startup protocol: {other:?}");
|
||||||
}
|
}
|
||||||
return Err(StartupTransportFailure(format!(
|
return Err(WorkerTransportFailure(format!(
|
||||||
"Gemma worker did not initialize: {other:?}"
|
"Gemma worker did not initialize: {other:?}"
|
||||||
))
|
))
|
||||||
.into());
|
.into());
|
||||||
@@ -203,6 +231,7 @@ impl GemmaEmbedder {
|
|||||||
Err(error) => {
|
Err(error) => {
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
let _ = child.wait();
|
let _ = child.wait();
|
||||||
|
let _ = reader_thread.join();
|
||||||
return Err(error).context("Invalid Gemma worker handshake");
|
return Err(error).context("Invalid Gemma worker handshake");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -216,11 +245,16 @@ impl GemmaEmbedder {
|
|||||||
{
|
{
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
let _ = child.wait();
|
let _ = child.wait();
|
||||||
|
let _ = reader_thread.join();
|
||||||
anyhow::bail!("Gemma worker handshake violates configured model/device contract");
|
anyhow::bail!("Gemma worker handshake violates configured model/device contract");
|
||||||
}
|
}
|
||||||
|
let (requests, writes, writer_thread) = spawn_writer(stdin);
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
child,
|
child,
|
||||||
stdin,
|
requests: Some(requests),
|
||||||
|
writes,
|
||||||
|
writer_thread: Some(writer_thread),
|
||||||
|
reader_thread: Some(reader_thread),
|
||||||
responses,
|
responses,
|
||||||
info,
|
info,
|
||||||
healthy: true,
|
healthy: true,
|
||||||
@@ -246,7 +280,10 @@ impl GemmaEmbedder {
|
|||||||
);
|
);
|
||||||
let response = match self.exchange(&encoded) {
|
let response = match self.exchange(&encoded) {
|
||||||
Ok(response) => response,
|
Ok(response) => response,
|
||||||
Err(error) if can_retry_cpu(&self.info.device, self.force_cpu) => {
|
Err(error)
|
||||||
|
if can_retry_cpu(&self.info.device, self.force_cpu)
|
||||||
|
&& error.downcast_ref::<WorkerTransportFailure>().is_some() =>
|
||||||
|
{
|
||||||
// GPU runtimes can abort rather than return an encoder error.
|
// GPU runtimes can abort rather than return an encoder error.
|
||||||
// Restart once as CPU and retry this same bounded request.
|
// Restart once as CPU and retry this same bounded request.
|
||||||
let _ = self.child.wait();
|
let _ = self.child.wait();
|
||||||
@@ -296,33 +333,76 @@ impl GemmaEmbedder {
|
|||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
|
fn stop_worker(&mut self) {
|
||||||
let write_started = self.query_profile.as_ref().map(|_| Instant::now());
|
self.requests.take();
|
||||||
let sent = (|| {
|
// The shipped native worker does not spawn descendants. Killing/reaping
|
||||||
self.stdin.write_all(encoded)?;
|
// it closes its pipe ends and releases any blocked writer/read thread.
|
||||||
self.stdin.write_all(b"\n")?;
|
|
||||||
self.stdin.flush()
|
|
||||||
})()
|
|
||||||
.context("Cannot write local Gemma worker request");
|
|
||||||
self.record_query_duration("request_write_us", write_started);
|
|
||||||
if let Err(error) = sent {
|
|
||||||
self.healthy = false;
|
|
||||||
self.info.ready = false;
|
|
||||||
let _ = self.child.kill();
|
let _ = self.child.kill();
|
||||||
return Err(error);
|
let _ = self.child.wait();
|
||||||
|
if let Some(writer) = self.writer_thread.take() {
|
||||||
|
let _ = writer.join();
|
||||||
}
|
}
|
||||||
let wait_started = self.query_profile.as_ref().map(|_| Instant::now());
|
if let Some(reader) = self.reader_thread.take() {
|
||||||
let response = match self.responses.recv_timeout(Duration::from_secs(180)) {
|
let _ = reader.join();
|
||||||
Ok(Ok(value)) => value,
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
|
||||||
|
self.exchange_with_timeout(encoded, Duration::from_secs(180))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exchange_with_timeout(&mut self, encoded: &[u8], timeout: Duration) -> Result<Value> {
|
||||||
|
let deadline = Instant::now() + timeout;
|
||||||
|
let write_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
|
let result = (|| {
|
||||||
|
self.requests
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| WorkerTransportFailure("Gemma worker writer stopped".into()))?
|
||||||
|
.try_send(encoded.to_vec())
|
||||||
|
.map_err(|_| {
|
||||||
|
WorkerTransportFailure("Gemma worker request queue unavailable".into())
|
||||||
|
})?;
|
||||||
|
match self
|
||||||
|
.writes
|
||||||
|
.recv_timeout(deadline.saturating_duration_since(Instant::now()))
|
||||||
|
{
|
||||||
|
Ok(Ok(())) => (),
|
||||||
other => {
|
other => {
|
||||||
self.healthy = false;
|
return Err(WorkerTransportFailure(format!(
|
||||||
self.info.ready = false;
|
"Gemma worker request write failed or timed out: {other:?}"
|
||||||
let _ = self.child.kill();
|
))
|
||||||
anyhow::bail!("Gemma worker request failed: {other:?}");
|
.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.record_query_duration("request_write_us", write_started);
|
||||||
|
let wait_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
|
let response = match self
|
||||||
|
.responses
|
||||||
|
.recv_timeout(deadline.saturating_duration_since(Instant::now()))
|
||||||
|
{
|
||||||
|
Ok(Ok(value)) => value,
|
||||||
|
Ok(Err(error))
|
||||||
|
if error.starts_with("Invalid Gemma worker JSON:")
|
||||||
|
|| error == "Gemma worker response exceeds protocol byte budget" =>
|
||||||
|
{
|
||||||
|
anyhow::bail!("Invalid Gemma worker response protocol: {error}")
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
return Err(WorkerTransportFailure(format!(
|
||||||
|
"Gemma worker request failed or timed out: {other:?}"
|
||||||
|
))
|
||||||
|
.into())
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
self.record_query_duration("worker_wait_and_response_decode_us", wait_started);
|
self.record_query_duration("worker_wait_and_response_decode_us", wait_started);
|
||||||
Ok(response)
|
Ok(response)
|
||||||
|
})();
|
||||||
|
if result.is_err() {
|
||||||
|
self.healthy = false;
|
||||||
|
self.info.ready = false;
|
||||||
|
self.stop_worker();
|
||||||
|
}
|
||||||
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>> {
|
pub fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>> {
|
||||||
@@ -710,6 +790,122 @@ fn verify_artifact(path: &Path, bytes: u64, sha: &str) -> Result<()> {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
// This helper is an inert pipe-protocol fixture, never an embedding provider.
|
||||||
|
fn stalled_protocol_helper(read_request: bool) -> GemmaEmbedder {
|
||||||
|
#[cfg(windows)]
|
||||||
|
let mut command = {
|
||||||
|
use std::os::windows::process::CommandExt;
|
||||||
|
let mut command = Command::new("powershell.exe");
|
||||||
|
command.args([
|
||||||
|
"-NoProfile",
|
||||||
|
"-NonInteractive",
|
||||||
|
"-Command",
|
||||||
|
if read_request {
|
||||||
|
"[Console]::In.ReadLine() | Out-Null; Start-Sleep -Seconds 20"
|
||||||
|
} else {
|
||||||
|
"Start-Sleep -Seconds 20"
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
command.creation_flags(0x08000000);
|
||||||
|
command
|
||||||
|
};
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
let mut command = {
|
||||||
|
let mut command = Command::new("sh");
|
||||||
|
command.args([
|
||||||
|
"-c",
|
||||||
|
if read_request {
|
||||||
|
"read line; exec sleep 20"
|
||||||
|
} else {
|
||||||
|
"exec sleep 20"
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
command
|
||||||
|
};
|
||||||
|
let mut child = command
|
||||||
|
.stdin(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::null())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
let (requests, writes, writer_thread) = spawn_writer(child.stdin.take().unwrap());
|
||||||
|
let stdout = child.stdout.take().unwrap();
|
||||||
|
let (sender, responses) = mpsc::sync_channel(1);
|
||||||
|
let reader_thread = thread::spawn(move || {
|
||||||
|
let response = read_response(&mut BufReader::new(stdout));
|
||||||
|
let _ = sender.try_send(response);
|
||||||
|
});
|
||||||
|
GemmaEmbedder {
|
||||||
|
child,
|
||||||
|
requests: Some(requests),
|
||||||
|
writes,
|
||||||
|
writer_thread: Some(writer_thread),
|
||||||
|
reader_thread: Some(reader_thread),
|
||||||
|
responses,
|
||||||
|
info: WorkerInfo {
|
||||||
|
ready: true,
|
||||||
|
model: "pipe protocol fixture".into(),
|
||||||
|
dimensions: 0,
|
||||||
|
device: "cpu".into(),
|
||||||
|
threads: 1,
|
||||||
|
runtime: "fixture".into(),
|
||||||
|
media_supported: false,
|
||||||
|
code_query_supported: false,
|
||||||
|
},
|
||||||
|
healthy: true,
|
||||||
|
force_cpu: true,
|
||||||
|
model_dir: PathBuf::new(),
|
||||||
|
worker_path: PathBuf::new(),
|
||||||
|
projector_verified: false,
|
||||||
|
query_profile: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn disconnected_writer_queue_is_transport_failure_and_reaps_helper() {
|
||||||
|
let mut fixture = stalled_protocol_helper(false);
|
||||||
|
let (sender, receiver) = mpsc::sync_channel(1);
|
||||||
|
drop(receiver);
|
||||||
|
fixture.requests = Some(sender);
|
||||||
|
let error = fixture
|
||||||
|
.exchange_with_timeout(b"{}", Duration::from_millis(300))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.downcast_ref::<WorkerTransportFailure>().is_some());
|
||||||
|
assert!(fixture.child.try_wait().unwrap().is_some());
|
||||||
|
assert!(!fixture.healthy && fixture.requests.is_none());
|
||||||
|
assert!(fixture.writer_thread.is_none() && fixture.reader_thread.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn entire_request_deadline_kills_and_joins_worker_that_never_reads_stdin() {
|
||||||
|
let mut fixture = stalled_protocol_helper(false);
|
||||||
|
let started = Instant::now();
|
||||||
|
let error = fixture
|
||||||
|
.exchange_with_timeout(&vec![b'x'; MAX_REQUEST - 1], Duration::from_millis(300))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.downcast_ref::<WorkerTransportFailure>().is_some());
|
||||||
|
assert!(error.to_string().contains("write failed or timed out"));
|
||||||
|
assert!(started.elapsed() < Duration::from_secs(5));
|
||||||
|
assert!(fixture.child.try_wait().unwrap().is_some());
|
||||||
|
assert!(!fixture.healthy && !fixture.info.ready);
|
||||||
|
assert!(fixture.writer_thread.is_none() && fixture.reader_thread.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn response_wait_uses_same_deadline_and_reaps_silent_reader() {
|
||||||
|
let mut fixture = stalled_protocol_helper(true);
|
||||||
|
let started = Instant::now();
|
||||||
|
let error = fixture
|
||||||
|
.exchange_with_timeout(b"{}", Duration::from_millis(300))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.downcast_ref::<WorkerTransportFailure>().is_some());
|
||||||
|
assert!(error.to_string().contains("request failed or timed out"));
|
||||||
|
assert!(started.elapsed() < Duration::from_secs(5));
|
||||||
|
assert!(fixture.child.try_wait().unwrap().is_some());
|
||||||
|
assert!(fixture.requests.is_none());
|
||||||
|
assert!(fixture.writer_thread.is_none() && fixture.reader_thread.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn automatic_startup_transport_failure_retries_cpu_once() {
|
fn automatic_startup_transport_failure_retries_cpu_once() {
|
||||||
let mut attempts = Vec::new();
|
let mut attempts = Vec::new();
|
||||||
@@ -718,7 +914,7 @@ mod tests {
|
|||||||
if cpu {
|
if cpu {
|
||||||
Ok("cpu ready")
|
Ok("cpu ready")
|
||||||
} else {
|
} else {
|
||||||
Err(StartupTransportFailure("worker closed its output before ready".into()).into())
|
Err(WorkerTransportFailure("worker closed its output before ready".into()).into())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -732,7 +928,7 @@ mod tests {
|
|||||||
let mut attempts = Vec::new();
|
let mut attempts = Vec::new();
|
||||||
let result: Result<()> = start_with_cpu_fallback(cpu, |selected_cpu| {
|
let result: Result<()> = start_with_cpu_fallback(cpu, |selected_cpu| {
|
||||||
attempts.push(selected_cpu);
|
attempts.push(selected_cpu);
|
||||||
Err(StartupTransportFailure("worker exited".into()).into())
|
Err(WorkerTransportFailure("worker exited".into()).into())
|
||||||
});
|
});
|
||||||
let error = format!("{:#}", result.unwrap_err());
|
let error = format!("{:#}", result.unwrap_err());
|
||||||
assert_eq!(attempts, if cpu { vec![true] } else { vec![false, true] });
|
assert_eq!(attempts, if cpu { vec![true] } else { vec![false, true] });
|
||||||
|
|||||||
@@ -2,6 +2,17 @@
|
|||||||
|
|
||||||
final result: browser captures checked; native desktop visual gate remains open
|
final result: browser captures checked; native desktop visual gate remains open
|
||||||
|
|
||||||
|
## Native release startup evidence (2026-10-12)
|
||||||
|
|
||||||
|
The extracted embedded-frontend Windows Release archive launches outside the
|
||||||
|
checkout with isolated validation data and CPU-only inference/rendering.
|
||||||
|
Accessibility confirms the `http://tauri.localhost/` frontend, one indexed file,
|
||||||
|
one extracted content segment and ready Google semantic search using CPU.
|
||||||
|
This verifies startup and engine status, not search interaction or appearance.
|
||||||
|
Native input fails with `GetCursorPos 0x80070005`; capture fails with
|
||||||
|
`CreateForMonitor 0x80070057`. No native screenshot was obtained. Native focus,
|
||||||
|
keyboard navigation, transitions and visual inspection remain open.
|
||||||
|
|
||||||
## Browser-rendered implementation evidence (2026-10-12)
|
## Browser-rendered implementation evidence (2026-10-12)
|
||||||
|
|
||||||
Four 1440×1000 captures now show the production frontend against the actual
|
Four 1440×1000 captures now show the production frontend against the actual
|
||||||
|
|||||||
+42
-7
@@ -35,8 +35,13 @@ cargo test -p searchi-service --test google_cpu -- --ignored --nocapture
|
|||||||
cargo test -p searchi-service --lib google_cpu_preserves -- --ignored --nocapture
|
cargo test -p searchi-service --lib google_cpu_preserves -- --ignored --nocapture
|
||||||
```
|
```
|
||||||
|
|
||||||
The worker is statically linked with the MSVC CRT on Windows. Staging copies
|
The explicit internal CPU worker uses the static MSVC CRT on Windows; normal
|
||||||
it and the llama.cpp license into the desktop resources directory. Add
|
auto builds use the shared runtime described below. Staging verifies a complete
|
||||||
|
candidate resource directory, then swaps it into place under a process lock.
|
||||||
|
Its recovery journal restores an interrupted swap on the next invocation;
|
||||||
|
failed restoration retains the previous backup. Existing unrelated fonts and
|
||||||
|
media resources are preserved. Fault-injected process interruption is tested;
|
||||||
|
power-loss durability is not certified. Add
|
||||||
`--bundle-model` to staging for an offline model bundle after downloading;
|
`--bundle-model` to staging for an offline model bundle after downloading;
|
||||||
otherwise desktop setup provisions verified models on first launch. Search
|
otherwise desktop setup provisions verified models on first launch. Search
|
||||||
itself performs no network inference or downloads. The worker now uses the verified media projector for genuine image/audio embeddings. The FFmpeg script stages verified standalone decoders and their notices.
|
itself performs no network inference or downloads. The worker now uses the verified media projector for genuine image/audio embeddings. The FFmpeg script stages verified standalone decoders and their notices.
|
||||||
@@ -59,25 +64,55 @@ supports `--backend auto|metal|vulkan|cuda` builds with CPU fallback; do not
|
|||||||
exercise those acceleration backends in this VM. Their execution and failure
|
exercise those acceleration backends in this VM. Their execution and failure
|
||||||
recovery require validation on the corresponding release hardware.
|
recovery require validation on the corresponding release hardware.
|
||||||
|
|
||||||
|
### Isolated native desktop validation
|
||||||
|
|
||||||
|
Set `SEARCHI_DATA_DIR` to an absolute directory to isolate the desktop's
|
||||||
|
settings and index for release QA or portable use. Normal launches use the OS
|
||||||
|
app-data directory. Invalid overrides fail startup instead of falling back to
|
||||||
|
the user's regular data. Model downloads still use the configured model
|
||||||
|
directory or managed cache, so isolated tests should also configure a verified
|
||||||
|
fixture model directory in their own settings.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$env:SEARCHI_DATA_DIR = 'C:\searchi-qa-data'
|
||||||
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
|
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
|
||||||
|
```
|
||||||
|
|
||||||
|
Extract and launch the entire Release archive outside the source checkout.
|
||||||
|
Direct Cargo release builds must embed the frontend:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cargo build --manifest-path apps/desktop/src-tauri/Cargo.toml --release --features custom-protocol --locked
|
||||||
|
```
|
||||||
|
|
||||||
|
The crate rejects Release compilation without this feature; the Tauri build
|
||||||
|
CLI enables it automatically. A development-server URL is not a release gate.
|
||||||
|
Record native window interaction separately from process startup, browser
|
||||||
|
screenshots and worker inference; those are different release gates.
|
||||||
|
|
||||||
## Headless Linux Google search
|
## Headless Linux Google search
|
||||||
|
|
||||||
These are the current script and CLI interfaces, not evidence of an executed
|
These are the current script and CLI interfaces, not evidence of an executed
|
||||||
Linux release. On a native Linux build host, run from the repository root:
|
Linux release. On a native Linux build host with Vulkan development libraries
|
||||||
|
and `glslc`, run from the repository root:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
python3 tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
python3 tools/embeddinggemma2/build_runtime.py --backend auto --jobs 2
|
||||||
python3 tools/embeddinggemma2/download.py
|
python3 tools/embeddinggemma2/download.py
|
||||||
python3 tools/media-runtime/provision.py
|
python3 tools/media-runtime/provision.py
|
||||||
export SEARCHI_FORCE_CPU=1
|
|
||||||
cargo build --release -p searchi-cli --locked
|
cargo build --release -p searchi-cli --locked
|
||||||
./target/release/searchi index --db searchi.sqlite --root /path/to/documents
|
./target/release/searchi index --db searchi.sqlite --root /path/to/documents
|
||||||
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
|
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/auto/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
|
||||||
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
|
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/auto/searchi-gemma-worker --cpu-threads 1
|
||||||
```
|
```
|
||||||
|
|
||||||
`embed` builds document, image, video and audio/music vectors by default. Use
|
`embed` builds document, image, video and audio/music vectors by default. Use
|
||||||
`--text-only` explicitly to omit media. Default media indexing needs paired
|
`--text-only` explicitly to omit media. Default media indexing needs paired
|
||||||
FFmpeg/FFprobe paths or packaged helpers beside the installed CLI.
|
FFmpeg/FFprobe paths or packaged helpers beside the installed CLI.
|
||||||
|
This normal worker detects usable acceleration and falls back to CPU. Internal
|
||||||
|
CPU-only model tests may use `--backend cpu` and `SEARCHI_FORCE_CPU=1`; those
|
||||||
|
test artifacts are not a separate distribution edition.
|
||||||
|
|
||||||
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
|
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
|
||||||
command. It provisions model assets, not the native worker. Package the worker
|
command. It provisions model assets, not the native worker. Package the worker
|
||||||
|
|||||||
+16
-6
@@ -67,11 +67,16 @@ Status: implementation started; no production release yet. Windows VM tests use
|
|||||||
- [x] Windows desktop archive packager includes explicit runtime/helper files
|
- [x] Windows desktop archive packager includes explicit runtime/helper files
|
||||||
and notices, checks architecture and records per-file checksums; excludes
|
and notices, checks architecture and records per-file checksums; excludes
|
||||||
model caches and user data. Controlled packaging regression tests pass.
|
model caches and user data. Controlled packaging regression tests pass.
|
||||||
- [ ] Verify embedded-frontend Release archive launch outside the source tree,
|
- [x] Embedded-frontend Windows Release archive launches outside the source tree
|
||||||
clean-machine prerequisites, GPU dependencies, upgrades and signing.
|
with isolated data and a ready real Google CPU semantic generation.
|
||||||
|
- [ ] Verify native interactions, clean-machine prerequisites, GPU dependencies,
|
||||||
|
upgrades and signing.
|
||||||
- [x] Preflight native license and the complete offline Google model pair before
|
- [x] Preflight native license and the complete offline Google model pair before
|
||||||
modifying staged resources; corrupted/missing assets preserve existing worker.
|
modifying staged resources; corrupted/missing assets preserve existing worker.
|
||||||
- [ ] Atomic resource staging and interrupted-copy recovery for release bundles.
|
- [x] Transactional resource staging, verified copied bytes, process locking,
|
||||||
|
interrupted-copy/swap recovery and backup preservation on failed rollback.
|
||||||
|
- [ ] Verify power-loss durability and release-tool interaction with resource
|
||||||
|
directory swaps on each OS; fault-injected process interruption is narrower.
|
||||||
- [x] Shared monotonic Basic/Debug search metrics, granular bounded engine traces,
|
- [x] Shared monotonic Basic/Debug search metrics, granular bounded engine traces,
|
||||||
actual worker query diagnostics, and headless JSON performance output.
|
actual worker query diagnostics, and headless JSON performance output.
|
||||||
- [ ] Verify detailed performance UI interaction on native desktop platforms;
|
- [ ] Verify detailed performance UI interaction on native desktop platforms;
|
||||||
@@ -84,7 +89,9 @@ Status: implementation started; no production release yet. Windows VM tests use
|
|||||||
Warm p95 alone does not satisfy the first-use requirement.
|
Warm p95 alone does not satisfy the first-use requirement.
|
||||||
- [ ] Test migrations, interrupted work, invalid documents, inaccessible/offline roots and cloud placeholders.
|
- [ ] Test migrations, interrupted work, invalid documents, inaccessible/offline roots and cloud placeholders.
|
||||||
- [ ] Automated native build matrix and documented supported cross-compilation paths.
|
- [ ] Automated native build matrix and documented supported cross-compilation paths.
|
||||||
- [ ] Execute Linux/macOS platform tests; create Windows/macOS/Linux installers.
|
- [x] Native Linux headless workspace tests: 93 passed, 11 explicitly ignored.
|
||||||
|
- [ ] Execute Linux native Google worker and desktop/macOS platform tests;
|
||||||
|
create Windows/macOS/Linux installers.
|
||||||
- [ ] Signing/notarization configuration, upgrade/rollback validation, release instructions.
|
- [ ] Signing/notarization configuration, upgrade/rollback validation, release instructions.
|
||||||
|
|
||||||
## 6. Local agents and optional answers
|
## 6. Local agents and optional answers
|
||||||
@@ -115,8 +122,11 @@ Audit and measured priorities: [EMBEDDINGGEMMA2-AUDIT.md](EMBEDDINGGEMMA2-AUDIT.
|
|||||||
preserve filtering, source freshness, failures and actual warmup probes.
|
preserve filtering, source freshness, failures and actual warmup probes.
|
||||||
- [x] Reduce semantic settle delay to 120 ms and retain old ready generation
|
- [x] Reduce semantic settle delay to 120 ms and retain old ready generation
|
||||||
during rebuilding; test initial-generation and activation readiness boundaries.
|
during rebuilding; test initial-generation and activation readiness boundaries.
|
||||||
- [ ] Bound whole IPC write/read deadlines and recover from a nonreading helper.
|
- [x] Bound whole IPC write/read deadlines and recover from a nonreading helper;
|
||||||
- [ ] Declare/test x86 CPU minimum or baseline/AVX2 runtime selection.
|
CPU-only real helper-process regressions cover blocked writes and responses.
|
||||||
|
- [x] Windows x64 baseline/optimized CPU runtime selection and OS AVX-state
|
||||||
|
checks, verified with real packaged baseline and optimized inference.
|
||||||
|
- [ ] Validate CPU selection on older hardware and remaining architectures.
|
||||||
- [ ] Incrementally reuse unchanged document/media vectors in atomic generations.
|
- [ ] Incrementally reuse unchanged document/media vectors in atomic generations.
|
||||||
- [ ] Benchmark genuine token-budgeted multi-sequence batching and isolation.
|
- [ ] Benchmark genuine token-budgeted multi-sequence batching and isolation.
|
||||||
- [ ] Implement and measure shared foreground/indexer resource budgets/priority.
|
- [ ] Implement and measure shared foreground/indexer resource budgets/priority.
|
||||||
|
|||||||
@@ -20,6 +20,12 @@ under the opt-in minilm-proof feature and tools/semantic-proof/HISTORICAL.md.
|
|||||||
The persistent native helper uses bounded NDJSON over stdin/stdout. It has no
|
The persistent native helper uses bounded NDJSON over stdin/stdout. It has no
|
||||||
HTTP server or download code. Rust verifies the text model's length and SHA256
|
HTTP server or download code. Rust verifies the text model's length and SHA256
|
||||||
before starting it. Setup provisions assets; search never downloads them.
|
before starting it. Setup provisions assets; search never downloads them.
|
||||||
|
Each request has one 180-second IPC deadline across pipe writing and response
|
||||||
|
reading. Single-slot queues bound pending work. A stalled write or response
|
||||||
|
kills/reaps the helper and joins both IPC threads; only transport failures
|
||||||
|
qualify for a bounded CPU restart. Malformed/oversized responses remain
|
||||||
|
protocol errors. Cleanup assumes the shipped helper spawns no descendants
|
||||||
|
holding its pipe handles, matching the native implementation.
|
||||||
|
|
||||||
Production device mode is auto, with available compiled acceleration and CPU
|
Production device mode is auto, with available compiled acceleration and CPU
|
||||||
fallback. Model/context initialization and returned inference/nonfinite-output
|
fallback. Model/context initialization and returned inference/nonfinite-output
|
||||||
|
|||||||
@@ -7,6 +7,39 @@ and `SEARCHI_FORCE_CPU=1`.
|
|||||||
|
|
||||||
## Current integration checkpoint (2026-10-12)
|
## Current integration checkpoint (2026-10-12)
|
||||||
|
|
||||||
|
The reliability checkpoint passes 93 workspace tests on Windows and 93 on
|
||||||
|
native Linux (Ubuntu 24.04 under WSL); each explicitly ignores 11 optional
|
||||||
|
integration tests. Linux uses its native Rust toolchain and an isolated ext4
|
||||||
|
build cache. This verifies headless core portability, not a Linux desktop,
|
||||||
|
Linux native Google worker, GPU or installer. Windows desktop passes 17 tests;
|
||||||
|
strict workspace and desktop Clippy checks pass. Five genuine Google model
|
||||||
|
integration tests pass in forced-CPU mode.
|
||||||
|
|
||||||
|
Worker IPC now applies one deadline across sending and receiving, with bounded
|
||||||
|
queues and owned writer/reader threads. Transport failures terminate and reap
|
||||||
|
the worker; eligible GPU transport failures retry on CPU. Invalid protocol
|
||||||
|
responses remain errors. Fifteen semantic unit tests pass, including a blocked
|
||||||
|
pipe and disconnected writer; one optional integration test is ignored.
|
||||||
|
|
||||||
|
Resource staging passes 24 tests on Windows (one symlink-permission skip) and
|
||||||
|
23 on Linux (two Windows-only skips). Locking, recovery, rollback preservation,
|
||||||
|
link rejection and concurrent edits before the final swap are covered. Actual
|
||||||
|
automatic-backend staging also succeeds. These are process-failure checks;
|
||||||
|
power-loss durability and concurrent arbitrary writes after the final check
|
||||||
|
remain release gates.
|
||||||
|
|
||||||
|
The latest unsigned Windows archive launches outside the checkout with an
|
||||||
|
embedded frontend at `http://tauri.localhost/`, isolated data, one real indexed
|
||||||
|
document and a ready Google CPU semantic generation. The archive is
|
||||||
|
145,217,175 bytes, SHA-256
|
||||||
|
`fb454a9dbf258f51044796ab24a5942a8f90804d0824ca0e2dc5f59b7847ec3b`.
|
||||||
|
Native accessibility confirms startup and status only. Input and screenshot
|
||||||
|
capture remain denied by the environment; native interaction, appearance,
|
||||||
|
clean-machine installation and GPU behavior are unverified. Release builds
|
||||||
|
without the embedded-frontend feature now fail explicitly.
|
||||||
|
|
||||||
|
The following package-dispatch evidence predates this reliability checkpoint:
|
||||||
|
|
||||||
The normal Windows x64 package now includes baseline and optimized CPU DLLs
|
The normal Windows x64 package now includes baseline and optimized CPU DLLs
|
||||||
with CPU/OS feature checks, plus the Vulkan backend. Shared DLLs use a common
|
with CPU/OS feature checks, plus the Vulkan backend. Shared DLLs use a common
|
||||||
MSVC CRT: a first `/MT` prototype crashed because model file handles crossed
|
MSVC CRT: a first `/MT` prototype crashed because model file handles crossed
|
||||||
|
|||||||
@@ -1,16 +1,327 @@
|
|||||||
"""Stage the native worker for installers; optional offline model bundle."""
|
"""Stage the native worker for installers; optional offline model bundle."""
|
||||||
import argparse
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import shutil
|
import shutil
|
||||||
|
import stat
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
|
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_link(path):
|
||||||
|
"""Reject Windows reparse points, including junctions, on Python 3.11+."""
|
||||||
|
path = Path(path)
|
||||||
|
if path.is_symlink():
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
attributes = getattr(path.lstat(), "st_file_attributes", 0)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return False
|
||||||
|
return bool(attributes & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
|
||||||
|
|
||||||
|
|
||||||
|
def _checked_path(path, parent):
|
||||||
|
"""Validate absolute workspace boundaries before any move or recursive delete."""
|
||||||
|
path, parent = Path(path).absolute(), Path(parent).resolve()
|
||||||
|
if _is_link(path) or path.parent.resolve() != parent:
|
||||||
|
raise RuntimeError("Staging path escapes its checked parent or is a link")
|
||||||
|
if path.exists() and path.resolve() != parent / path.name:
|
||||||
|
raise RuntimeError("Staging path resolves outside its checked parent")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _check_tree(directory):
|
||||||
|
if _is_link(directory):
|
||||||
|
raise RuntimeError("Resource staging refuses symlinks or junctions")
|
||||||
|
count = 0
|
||||||
|
for parent, directories, files in os.walk(directory, followlinks=False):
|
||||||
|
for name in [*directories, *files]:
|
||||||
|
count += 1
|
||||||
|
if count > 100000:
|
||||||
|
raise RuntimeError("Resource staging exceeds its file-count limit")
|
||||||
|
path = Path(parent) / name
|
||||||
|
if _is_link(path) or not path.resolve().is_relative_to(directory.resolve()):
|
||||||
|
raise RuntimeError("Resource staging refuses symlinks or junctions")
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_tree(path, parent):
|
||||||
|
checked = _checked_path(path, parent)
|
||||||
|
if checked.exists():
|
||||||
|
_check_tree(checked)
|
||||||
|
shutil.rmtree(checked)
|
||||||
|
|
||||||
|
|
||||||
|
def _move_directory(source, destination, parent):
|
||||||
|
"""Bounded retry for transient Windows sharing locks during directory swaps."""
|
||||||
|
for attempt in range(4):
|
||||||
|
source = _checked_path(source, parent)
|
||||||
|
destination = _checked_path(destination, parent)
|
||||||
|
try:
|
||||||
|
os.replace(source, destination)
|
||||||
|
return
|
||||||
|
except PermissionError:
|
||||||
|
if attempt == 3:
|
||||||
|
raise
|
||||||
|
time.sleep(0.05 * (2 ** attempt))
|
||||||
|
|
||||||
|
|
||||||
|
def _record(path):
|
||||||
|
path = Path(path)
|
||||||
|
if _is_link(path) or path.absolute() != path.resolve() or not path.is_file():
|
||||||
|
raise RuntimeError(f"Staging source is missing or is a link: {path.name}")
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
return path.stat().st_size, hashlib.file_digest(stream, "sha256").hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _verified_copy(source, destination, expected=None):
|
||||||
|
source, destination = Path(source), Path(destination)
|
||||||
|
expected = expected or _record(source)
|
||||||
|
if destination.absolute() != destination.resolve():
|
||||||
|
raise RuntimeError("Staging copy target resolves through a link")
|
||||||
|
shutil.copy2(source, destination)
|
||||||
|
if _record(destination) != expected:
|
||||||
|
raise RuntimeError(f"Staging source changed or copy was incomplete: {source.name}")
|
||||||
|
return str(destination)
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def staging_lock(parent):
|
||||||
|
"""Kernel-held lock releases after process crashes; PID is informational."""
|
||||||
|
path = _checked_path(parent / ".searchi-stage.lock", parent)
|
||||||
|
with path.open("a+b") as stream:
|
||||||
|
stream.seek(0, os.SEEK_END)
|
||||||
|
if stream.tell() == 0:
|
||||||
|
stream.write(b"\0")
|
||||||
|
stream.flush()
|
||||||
|
stream.seek(0)
|
||||||
|
try:
|
||||||
|
if os.name == "nt":
|
||||||
|
import msvcrt
|
||||||
|
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||||
|
else:
|
||||||
|
import fcntl
|
||||||
|
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except OSError as error:
|
||||||
|
raise RuntimeError("Another resource staging process holds the lock") from error
|
||||||
|
try:
|
||||||
|
stream.seek(1)
|
||||||
|
stream.truncate()
|
||||||
|
stream.write(f"pid={os.getpid()}".encode())
|
||||||
|
stream.flush()
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
stream.seek(0)
|
||||||
|
if os.name == "nt":
|
||||||
|
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
||||||
|
else:
|
||||||
|
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
def _journal_paths(destination, state):
|
||||||
|
if (not isinstance(state, dict) or state.get("schema") != 1
|
||||||
|
or state.get("owner") != "searchi-runtime-staging"
|
||||||
|
or not isinstance(state.get("transaction"), str)
|
||||||
|
or not re.fullmatch(r"[0-9a-f]{32}", state["transaction"])
|
||||||
|
or type(state.get("had_destination")) is not bool):
|
||||||
|
raise RuntimeError("Invalid resource staging recovery journal")
|
||||||
|
parent = destination.parent.resolve()
|
||||||
|
transaction = state["transaction"]
|
||||||
|
return (_checked_path(parent / f".resources.stage-{transaction}", parent),
|
||||||
|
_checked_path(parent / f".resources.backup-{transaction}", parent))
|
||||||
|
|
||||||
|
|
||||||
|
def recover_staging(destination):
|
||||||
|
"""Recover interrupted copy/swap while caller holds the staging lock.
|
||||||
|
|
||||||
|
This handles process interruption, not certified power-loss durability.
|
||||||
|
An unsuccessful restoration retains the journal and original backup.
|
||||||
|
"""
|
||||||
|
parent = destination.parent.resolve()
|
||||||
|
destination = _checked_path(destination, parent)
|
||||||
|
journal = _checked_path(parent / ".searchi-stage.json", parent)
|
||||||
|
if not journal.exists():
|
||||||
|
return
|
||||||
|
if journal.stat().st_size > 4096:
|
||||||
|
raise RuntimeError("Oversized resource staging recovery journal")
|
||||||
|
state = json.loads(journal.read_text(encoding="utf-8"))
|
||||||
|
candidate, backup = _journal_paths(destination, state)
|
||||||
|
for directory in [destination, candidate, backup]:
|
||||||
|
if directory.exists():
|
||||||
|
_check_tree(directory)
|
||||||
|
marker = ".searchi-stage-complete"
|
||||||
|
|
||||||
|
def complete(directory):
|
||||||
|
path = directory / marker
|
||||||
|
return path.is_file() and path.stat().st_size == 32 and path.read_text() == state["transaction"]
|
||||||
|
|
||||||
|
if not destination.exists():
|
||||||
|
if backup.exists():
|
||||||
|
# If restoration fails, do not discard either recoverable directory.
|
||||||
|
_move_directory(backup, destination, parent)
|
||||||
|
elif candidate.exists() and complete(candidate):
|
||||||
|
_move_directory(candidate, destination, parent)
|
||||||
|
elif state["had_destination"]:
|
||||||
|
raise RuntimeError("Original staging directory and backup are both missing; recovery retained")
|
||||||
|
if destination.exists() and backup.exists():
|
||||||
|
if not complete(destination):
|
||||||
|
raise RuntimeError("Ambiguous resource staging recovery; original backup retained")
|
||||||
|
_remove_tree(backup, parent)
|
||||||
|
_remove_tree(candidate, parent)
|
||||||
|
if destination.exists():
|
||||||
|
completion = destination / marker
|
||||||
|
if completion.exists():
|
||||||
|
if not complete(destination):
|
||||||
|
raise RuntimeError("Invalid resource staging completion marker")
|
||||||
|
completion.unlink()
|
||||||
|
journal.unlink()
|
||||||
|
|
||||||
|
|
||||||
|
def _previous_owned(destination):
|
||||||
|
names = {"searchi-gemma-worker.exe", "searchi-gemma-worker", "runtime-libraries.json",
|
||||||
|
"LLAMA-LICENSE.txt", "VULKAN-LICENSE.txt", "MSVC-REDISTRIBUTION.txt", "vulkan-1.dll"}
|
||||||
|
manifest_path = Path(__file__).with_name("model-manifest.json")
|
||||||
|
if manifest_path.stat().st_size > 65536:
|
||||||
|
raise RuntimeError("Pinned model manifest is oversized")
|
||||||
|
model_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
files = model_manifest.get("files") if isinstance(model_manifest, dict) else None
|
||||||
|
if not isinstance(files, list) or not 1 <= len(files) <= 4:
|
||||||
|
raise RuntimeError("Invalid pinned model manifest")
|
||||||
|
for record in files:
|
||||||
|
name = record.get("path") if isinstance(record, dict) else None
|
||||||
|
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.gguf", name):
|
||||||
|
raise RuntimeError("Invalid pinned model resource path")
|
||||||
|
names.add(f"embeddinggemma2/{name}")
|
||||||
|
inventory = destination / "runtime-libraries.json"
|
||||||
|
if inventory.exists():
|
||||||
|
if inventory.stat().st_size > 65536:
|
||||||
|
raise RuntimeError("Previous runtime inventory is oversized")
|
||||||
|
previous = json.loads(inventory.read_text(encoding="utf-8"))
|
||||||
|
records = previous.get("libraries") if isinstance(previous, dict) else None
|
||||||
|
if not isinstance(records, list) or len(records) > 64:
|
||||||
|
raise RuntimeError("Invalid previous runtime inventory")
|
||||||
|
for record in records:
|
||||||
|
name = record.get("path") if isinstance(record, dict) else None
|
||||||
|
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+\.dll", name):
|
||||||
|
raise RuntimeError("Invalid previous runtime library path")
|
||||||
|
names.add(name)
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def _resource_key(path):
|
||||||
|
value = Path(path).as_posix()
|
||||||
|
return value.casefold() if os.name == "nt" else value
|
||||||
|
|
||||||
|
|
||||||
|
def _preserved_snapshot(destination, owned):
|
||||||
|
"""Bounded content/tree snapshot detects edits to resources being retained."""
|
||||||
|
if not destination.exists():
|
||||||
|
return None
|
||||||
|
_check_tree(destination)
|
||||||
|
snapshot = {}
|
||||||
|
count = 0
|
||||||
|
for parent, directories, files in os.walk(destination, followlinks=False):
|
||||||
|
for name in [*directories, *files]:
|
||||||
|
count += 1
|
||||||
|
if count > 100000:
|
||||||
|
raise RuntimeError("Resource staging exceeds its snapshot file-count limit")
|
||||||
|
path = Path(parent) / name
|
||||||
|
key = _resource_key(path.relative_to(destination))
|
||||||
|
if key in owned:
|
||||||
|
if not path.is_file():
|
||||||
|
raise RuntimeError("Owned runtime resource is not a file")
|
||||||
|
continue
|
||||||
|
if path.is_dir():
|
||||||
|
snapshot[key] = ("directory",)
|
||||||
|
else:
|
||||||
|
record = _record(path)
|
||||||
|
metadata = path.stat()
|
||||||
|
snapshot[key] = ("file", *record, metadata.st_mtime_ns, metadata.st_ino, metadata.st_dev)
|
||||||
|
return snapshot
|
||||||
|
|
||||||
|
|
||||||
|
def stage_resources(destination, sources, validate=None):
|
||||||
|
"""Replace a verified complete resource directory and preserve other assets."""
|
||||||
|
destination = Path(destination).absolute()
|
||||||
|
if destination.parent != destination.parent.resolve():
|
||||||
|
raise RuntimeError("Staging parent resolves through a link outside its expected location")
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
parent = destination.parent.resolve()
|
||||||
|
destination = _checked_path(destination, parent)
|
||||||
|
with staging_lock(parent):
|
||||||
|
recover_staging(destination)
|
||||||
|
if destination.exists():
|
||||||
|
_check_tree(destination)
|
||||||
|
if (destination / ".searchi-stage-complete").exists():
|
||||||
|
raise RuntimeError("Reserved staging completion marker exists without its journal")
|
||||||
|
owned = _previous_owned(destination)
|
||||||
|
owned_keys = {_resource_key(path) for path in owned}
|
||||||
|
preserved = _preserved_snapshot(destination, owned_keys)
|
||||||
|
records = []
|
||||||
|
targets = set()
|
||||||
|
for source, relative in sources:
|
||||||
|
relative = Path(relative)
|
||||||
|
if relative.is_absolute() or relative.anchor or relative.drive or ".." in relative.parts or not relative.parts:
|
||||||
|
raise RuntimeError("Invalid staging resource target")
|
||||||
|
key = relative.as_posix().lower()
|
||||||
|
if key in targets:
|
||||||
|
raise RuntimeError("Duplicate staging resource target")
|
||||||
|
targets.add(key)
|
||||||
|
records.append((Path(source), relative, _record(source)))
|
||||||
|
state = {"schema": 1, "owner": "searchi-runtime-staging", "transaction": uuid.uuid4().hex,
|
||||||
|
"had_destination": destination.exists()}
|
||||||
|
candidate, backup = _journal_paths(destination, state)
|
||||||
|
journal = _checked_path(parent / ".searchi-stage.json", parent)
|
||||||
|
pending = _checked_path(parent / f".searchi-stage-journal-{state['transaction']}.pending", parent)
|
||||||
|
try:
|
||||||
|
with pending.open("x", encoding="utf-8") as stream:
|
||||||
|
json.dump(state, stream)
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
os.replace(pending, journal)
|
||||||
|
finally:
|
||||||
|
pending.unlink(missing_ok=True)
|
||||||
|
try:
|
||||||
|
if destination.exists():
|
||||||
|
def ignore_owned(directory, names):
|
||||||
|
relative = Path(directory).relative_to(destination)
|
||||||
|
return [name for name in names if _resource_key(relative / name) in owned_keys]
|
||||||
|
|
||||||
|
shutil.copytree(destination, candidate, copy_function=_verified_copy, ignore=ignore_owned)
|
||||||
|
else:
|
||||||
|
candidate.mkdir()
|
||||||
|
model_directory = candidate / "embeddinggemma2"
|
||||||
|
if model_directory.is_dir() and not any(model_directory.iterdir()):
|
||||||
|
if not model_directory.resolve().is_relative_to(candidate.resolve()):
|
||||||
|
raise RuntimeError("Owned model directory escapes candidate")
|
||||||
|
model_directory.rmdir()
|
||||||
|
for source, relative, expected in records:
|
||||||
|
target = candidate / relative
|
||||||
|
if not target.resolve().is_relative_to(candidate.resolve()):
|
||||||
|
raise RuntimeError("Staging resource target escapes candidate directory")
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
_verified_copy(source, target, expected)
|
||||||
|
_check_tree(candidate)
|
||||||
|
if validate:
|
||||||
|
validate(candidate)
|
||||||
|
if _preserved_snapshot(destination, owned_keys) != preserved:
|
||||||
|
raise RuntimeError("Preserved resources changed during staging; original retained")
|
||||||
|
(candidate / ".searchi-stage-complete").write_text(state["transaction"])
|
||||||
|
if destination.exists():
|
||||||
|
_move_directory(destination, backup, parent)
|
||||||
|
_move_directory(candidate, destination, parent)
|
||||||
|
recover_staging(destination)
|
||||||
|
except Exception:
|
||||||
|
recover_staging(destination)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
def verify_runtime(source, required=False):
|
def verify_runtime(source, required=False):
|
||||||
"""Preflight an explicit, bounded dependency inventory, never a DLL glob."""
|
"""Preflight an explicit, bounded dependency inventory, never a DLL glob."""
|
||||||
inventory = source.parent / "runtime-libraries.json"
|
inventory = source.parent / "runtime-libraries.json"
|
||||||
@@ -74,6 +385,12 @@ def main():
|
|||||||
parser.add_argument("--backend", choices=["cpu", "auto", "metal", "vulkan", "cuda"], default="auto")
|
parser.add_argument("--backend", choices=["cpu", "auto", "metal", "vulkan", "cuda"], default="auto")
|
||||||
parser.add_argument("--bundle-model", action="store_true")
|
parser.add_argument("--bundle-model", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
destination = ROOT / "apps/desktop/src-tauri/resources"
|
||||||
|
if destination.parent.absolute() != destination.parent.resolve():
|
||||||
|
raise RuntimeError("Staging parent resolves through a link outside its expected location")
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with staging_lock(destination.parent):
|
||||||
|
recover_staging(destination)
|
||||||
name = "searchi-gemma-worker.exe" if os.name == "nt" else "searchi-gemma-worker"
|
name = "searchi-gemma-worker.exe" if os.name == "nt" else "searchi-gemma-worker"
|
||||||
source = ROOT / ".searchi/runtime" / args.backend
|
source = ROOT / ".searchi/runtime" / args.backend
|
||||||
if os.name == "nt":
|
if os.name == "nt":
|
||||||
@@ -91,29 +408,30 @@ def main():
|
|||||||
raise RuntimeError("Microsoft CRT redistribution notice is missing")
|
raise RuntimeError("Microsoft CRT redistribution notice is missing")
|
||||||
dependencies.append(crt_notice)
|
dependencies.append(crt_notice)
|
||||||
artifacts = []
|
artifacts = []
|
||||||
|
model_manifest = None
|
||||||
if args.bundle_model:
|
if args.bundle_model:
|
||||||
manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
model_manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||||
artifacts = verify_bundle(ROOT, manifest)
|
artifacts = verify_bundle(ROOT, model_manifest)
|
||||||
loader = None
|
loader = None
|
||||||
if os.name == "nt" and args.backend in {"auto", "vulkan"}:
|
if os.name == "nt" and args.backend in {"auto", "vulkan"}:
|
||||||
from provision_vulkan import provision
|
from provision_vulkan import provision
|
||||||
_, loader = provision()
|
_, loader = provision()
|
||||||
destination = ROOT / "apps/desktop/src-tauri/resources"
|
sources = [(source, source.name), (license_path, "LLAMA-LICENSE.txt")]
|
||||||
destination.mkdir(parents=True, exist_ok=True)
|
sources.extend((path, path.name) for path in dependencies)
|
||||||
shutil.copy2(source, destination / name)
|
|
||||||
for dependency in dependencies:
|
|
||||||
shutil.copy2(dependency, destination / dependency.name)
|
|
||||||
# Include the upstream runtime license with the statically linked helper.
|
|
||||||
shutil.copy2(license_path, destination / "LLAMA-LICENSE.txt")
|
|
||||||
if loader:
|
if loader:
|
||||||
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
|
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
|
||||||
if name not in {path.name for path in dependencies}:
|
if name not in {path.name for path in dependencies}:
|
||||||
shutil.copy2(loader / name, destination / name)
|
sources.append((loader / name, name))
|
||||||
if args.bundle_model:
|
sources.extend((path, f"embeddinggemma2/{path.name}") for path in artifacts)
|
||||||
model_destination = destination / "embeddinggemma2"
|
|
||||||
model_destination.mkdir(exist_ok=True)
|
def validate(candidate):
|
||||||
for artifact in artifacts:
|
verify_runtime(candidate / source.name, os.name == "nt" and args.backend in {"auto", "vulkan"})
|
||||||
shutil.copy2(artifact, model_destination / artifact.name)
|
if model_manifest:
|
||||||
|
for artifact, spec in zip(artifacts, model_manifest["files"], strict=True):
|
||||||
|
if _record(candidate / "embeddinggemma2" / artifact.name) != (spec["bytes"], spec["sha256"]):
|
||||||
|
raise RuntimeError("Copied model checksum mismatch")
|
||||||
|
|
||||||
|
stage_resources(destination, sources, validate)
|
||||||
print(f"Staged native worker: {destination / source.name}")
|
print(f"Staged native worker: {destination / source.name}")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
@@ -11,6 +12,371 @@ import stage_runtime
|
|||||||
|
|
||||||
|
|
||||||
class StagingTests(unittest.TestCase):
|
class StagingTests(unittest.TestCase):
|
||||||
|
def resources(self, directory):
|
||||||
|
root = Path(directory)
|
||||||
|
destination = root / "src-tauri/resources"
|
||||||
|
destination.mkdir(parents=True)
|
||||||
|
(destination / "searchi-gemma-worker.exe").write_bytes(b"previous worker")
|
||||||
|
(destination / "fonts").mkdir()
|
||||||
|
(destination / "fonts/Inter.woff2").write_bytes(b"font")
|
||||||
|
(destination / "media-runtime").mkdir()
|
||||||
|
(destination / "media-runtime/ffmpeg.exe").write_bytes(b"media")
|
||||||
|
(destination / "user-note.txt").write_bytes(b"preserve user file")
|
||||||
|
source = root / "worker.exe"
|
||||||
|
source.write_bytes(b"next worker")
|
||||||
|
return destination, source
|
||||||
|
|
||||||
|
def test_failed_copy_preserves_previous_complete_resources(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.shutil.copy2
|
||||||
|
|
||||||
|
def fail_worker(path, target, *args, **kwargs):
|
||||||
|
if Path(path) == source:
|
||||||
|
raise OSError("injected copy failure")
|
||||||
|
return original(path, target, *args, **kwargs)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.shutil, "copy2", fail_worker):
|
||||||
|
with self.assertRaisesRegex(OSError, "copy failure"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
self.assertEqual((destination / "user-note.txt").read_bytes(), b"preserve user file")
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
self.assertEqual(list(destination.parent.glob(".resources.stage-*")), [])
|
||||||
|
|
||||||
|
def test_changed_copy_bytes_reject_before_swap(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.shutil.copy2
|
||||||
|
|
||||||
|
def corrupt_worker(path, target, *args, **kwargs):
|
||||||
|
result = original(path, target, *args, **kwargs)
|
||||||
|
if Path(path) == source:
|
||||||
|
Path(target).write_bytes(b"bad worker!")
|
||||||
|
return result
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.shutil, "copy2", corrupt_worker):
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "changed or copy was incomplete"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
|
||||||
|
def test_source_change_after_preflight_rejects_before_swap(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.shutil.copy2
|
||||||
|
|
||||||
|
def mutate_source(path, target, *args, **kwargs):
|
||||||
|
if Path(path) == source:
|
||||||
|
source.write_bytes(b"evil worker")
|
||||||
|
return original(path, target, *args, **kwargs)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.shutil, "copy2", mutate_source):
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "changed or copy was incomplete"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
|
||||||
|
def test_resource_targets_reject_traversal_and_windows_drive_roots(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
targets = ["../outside"]
|
||||||
|
if os.name == "nt":
|
||||||
|
targets.extend(["C:outside", "\\outside"])
|
||||||
|
for target in targets:
|
||||||
|
with self.subTest(target=target), self.assertRaisesRegex(RuntimeError, "resource target"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, target)])
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
|
||||||
|
def test_locked_destination_rename_preserves_old_resources(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.os.replace
|
||||||
|
|
||||||
|
def fail_old_rename(path, target):
|
||||||
|
if Path(path) == destination:
|
||||||
|
raise PermissionError("destination is locked")
|
||||||
|
return original(path, target)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.os, "replace", fail_old_rename):
|
||||||
|
with self.assertRaisesRegex(PermissionError, "locked"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
|
||||||
|
def test_interrupted_between_renames_restores_old_before_source_preflight(self):
|
||||||
|
class Interrupt(BaseException):
|
||||||
|
pass
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.os.replace
|
||||||
|
|
||||||
|
def interrupt_candidate(path, target):
|
||||||
|
if Path(path).name.startswith(".resources.stage-"):
|
||||||
|
raise Interrupt()
|
||||||
|
return original(path, target)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.os, "replace", interrupt_candidate):
|
||||||
|
with self.assertRaises(Interrupt):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertFalse(destination.exists())
|
||||||
|
self.assertEqual(len(list(destination.parent.glob(".resources.backup-*"))), 1)
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "source is missing"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
|
||||||
|
def test_interrupted_after_install_commits_verified_new_resources(self):
|
||||||
|
class Interrupt(BaseException):
|
||||||
|
pass
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.os.replace
|
||||||
|
|
||||||
|
def interrupt_after_candidate(path, target):
|
||||||
|
original(path, target)
|
||||||
|
if Path(path).name.startswith(".resources.stage-"):
|
||||||
|
raise Interrupt()
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.os, "replace", interrupt_after_candidate):
|
||||||
|
with self.assertRaises(Interrupt):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "source is missing"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"next worker")
|
||||||
|
self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
|
||||||
|
self.assertEqual((destination / "media-runtime/ffmpeg.exe").read_bytes(), b"media")
|
||||||
|
self.assertEqual((destination / "user-note.txt").read_bytes(), b"preserve user file")
|
||||||
|
self.assertEqual(list(destination.parent.glob(".resources.backup-*")), [])
|
||||||
|
self.assertFalse((destination / ".searchi-stage-complete").exists())
|
||||||
|
|
||||||
|
def test_failed_rollback_retains_backup_and_journal_for_retry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.os.replace
|
||||||
|
|
||||||
|
def fail_candidate_and_restore(path, target):
|
||||||
|
if Path(path).name.startswith((".resources.stage-", ".resources.backup-")):
|
||||||
|
raise PermissionError("injected rename failure")
|
||||||
|
return original(path, target)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.os, "replace", fail_candidate_and_restore):
|
||||||
|
with self.assertRaisesRegex(PermissionError, "rename failure"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
backup = next(destination.parent.glob(".resources.backup-*"))
|
||||||
|
self.assertEqual((backup / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
self.assertTrue((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "source is missing"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
|
||||||
|
def test_profile_switch_removes_only_previous_inventory_libraries(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
old = destination / "ggml-cpu-old.dll"
|
||||||
|
old.write_bytes(b"old backend")
|
||||||
|
(destination / "user-plugin.dll").write_bytes(b"user plugin")
|
||||||
|
(destination / "runtime-libraries.json").write_text(json.dumps({"libraries": [{"path": old.name}]}))
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertFalse((destination / old.name).exists())
|
||||||
|
self.assertFalse((destination / "runtime-libraries.json").exists())
|
||||||
|
self.assertEqual((destination / "user-plugin.dll").read_bytes(), b"user plugin")
|
||||||
|
self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
|
||||||
|
|
||||||
|
def test_offline_to_online_removes_only_exact_pinned_model_pair(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
model_directory = destination / "embeddinggemma2"
|
||||||
|
model_directory.mkdir()
|
||||||
|
manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
|
||||||
|
for spec in manifest["files"]:
|
||||||
|
(model_directory / spec["path"]).write_bytes(b"old unverified model")
|
||||||
|
(model_directory / "user-note.txt").write_bytes(b"preserve unrelated asset")
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
for spec in manifest["files"]:
|
||||||
|
self.assertFalse((destination / "embeddinggemma2" / spec["path"]).exists())
|
||||||
|
self.assertEqual((destination / "embeddinggemma2/user-note.txt").read_bytes(), b"preserve unrelated asset")
|
||||||
|
self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
|
||||||
|
|
||||||
|
def test_online_staging_removes_empty_owned_model_directory(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
model_directory = destination / "embeddinggemma2"
|
||||||
|
model_directory.mkdir()
|
||||||
|
manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
|
||||||
|
for spec in manifest["files"]:
|
||||||
|
(model_directory / spec["path"]).write_bytes(b"old model")
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertFalse((destination / "embeddinggemma2").exists())
|
||||||
|
|
||||||
|
def test_old_owned_worker_and_model_pair_are_never_copied(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
model_directory = destination / "embeddinggemma2"
|
||||||
|
model_directory.mkdir()
|
||||||
|
manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
|
||||||
|
old_paths = {destination / "searchi-gemma-worker.exe"}
|
||||||
|
for spec in manifest["files"]:
|
||||||
|
path = model_directory / spec["path"]
|
||||||
|
path.write_bytes(b"old owned model")
|
||||||
|
old_paths.add(path)
|
||||||
|
original = stage_runtime.shutil.copy2
|
||||||
|
|
||||||
|
original_record = stage_runtime._record
|
||||||
|
|
||||||
|
def reject_owned_copy(path, target, *args, **kwargs):
|
||||||
|
if Path(path) in old_paths:
|
||||||
|
self.fail("Previously owned runtime/model asset was unnecessarily copied")
|
||||||
|
return original(path, target, *args, **kwargs)
|
||||||
|
|
||||||
|
def reject_owned_hash(path):
|
||||||
|
if Path(path) in old_paths:
|
||||||
|
self.fail("Previously owned runtime/model asset was unnecessarily hashed")
|
||||||
|
return original_record(path)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.shutil, "copy2", reject_owned_copy), patch.object(stage_runtime, "_record", reject_owned_hash):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"next worker")
|
||||||
|
self.assertFalse((destination / "embeddinggemma2").exists())
|
||||||
|
|
||||||
|
def test_concurrent_preserved_edit_aborts_and_keeps_latest_original(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
|
||||||
|
def edit_original(_candidate):
|
||||||
|
(destination / "user-note.txt").write_bytes(b"latest user edit")
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "Preserved resources changed"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")], edit_original)
|
||||||
|
self.assertEqual((destination / "user-note.txt").read_bytes(), b"latest user edit")
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
|
||||||
|
def test_concurrent_preserved_addition_aborts_and_keeps_new_file(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
|
||||||
|
def add_original(_candidate):
|
||||||
|
(destination / "fonts/new-font.woff2").write_bytes(b"new user asset")
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "Preserved resources changed"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")], add_original)
|
||||||
|
self.assertEqual((destination / "fonts/new-font.woff2").read_bytes(), b"new user asset")
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
|
||||||
|
def test_concurrent_staging_lock_fails_without_mutation(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
with stage_runtime.staging_lock(destination.parent):
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "holds the lock"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
|
||||||
|
def test_malicious_recovery_journal_cannot_escape_sibling_paths(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
outside = Path(directory) / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
(outside / "user-data.txt").write_bytes(b"never remove")
|
||||||
|
journal = destination.parent / ".searchi-stage.json"
|
||||||
|
journal.write_text(json.dumps({"schema": 1, "owner": "searchi-runtime-staging", "transaction": "../outside", "had_destination": True}))
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "recovery journal"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((outside / "user-data.txt").read_bytes(), b"never remove")
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
|
||||||
|
def test_existing_unrelated_symlink_rejects_before_stage_changes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
outside = Path(directory) / "outside.txt"
|
||||||
|
outside.write_bytes(b"private data")
|
||||||
|
link = destination / "user-link.txt"
|
||||||
|
try:
|
||||||
|
link.symlink_to(outside)
|
||||||
|
except OSError:
|
||||||
|
self.skipTest("Host does not permit creating test symlinks")
|
||||||
|
try:
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "symlinks or junctions"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual(outside.read_bytes(), b"private data")
|
||||||
|
self.assertTrue(link.is_symlink())
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
finally:
|
||||||
|
link.unlink()
|
||||||
|
|
||||||
|
def test_first_install_interrupt_after_verified_candidate_recovers(self):
|
||||||
|
class Interrupt(BaseException):
|
||||||
|
pass
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
destination = root / "src-tauri/resources"
|
||||||
|
source = root / "worker.exe"
|
||||||
|
source.write_bytes(b"first worker")
|
||||||
|
original = stage_runtime.os.replace
|
||||||
|
|
||||||
|
def interrupt_candidate(path, target):
|
||||||
|
if Path(path).name.startswith(".resources.stage-"):
|
||||||
|
raise Interrupt()
|
||||||
|
return original(path, target)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.os, "replace", interrupt_candidate):
|
||||||
|
with self.assertRaises(Interrupt):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertFalse(destination.exists())
|
||||||
|
with stage_runtime.staging_lock(destination.parent):
|
||||||
|
stage_runtime.recover_staging(destination)
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"first worker")
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.name == "nt", "Windows junction boundary regression")
|
||||||
|
def test_existing_junction_rejects_without_following_outside_directory(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
outside = Path(directory) / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
(outside / "private.txt").write_bytes(b"untouched")
|
||||||
|
link = destination / "user-junction"
|
||||||
|
result = subprocess.run(["cmd.exe", "/c", "mklink", "/J", str(link), str(outside)], capture_output=True)
|
||||||
|
if result.returncode:
|
||||||
|
self.skipTest("Host does not permit creating a test junction")
|
||||||
|
try:
|
||||||
|
self.assertTrue(stage_runtime._is_link(link))
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "symlinks or junctions"):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((outside / "private.txt").read_bytes(), b"untouched")
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
finally:
|
||||||
|
# Remove only the verified test junction, never its target tree.
|
||||||
|
self.assertEqual(link.absolute().parent, destination.absolute())
|
||||||
|
self.assertTrue(stage_runtime._is_link(link))
|
||||||
|
link.rmdir()
|
||||||
|
|
||||||
|
def test_interrupted_partial_candidate_keeps_old_resources_and_recovers(self):
|
||||||
|
class Interrupt(BaseException):
|
||||||
|
pass
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
destination, source = self.resources(directory)
|
||||||
|
original = stage_runtime.shutil.copy2
|
||||||
|
|
||||||
|
def interrupt_copy(path, target, *args, **kwargs):
|
||||||
|
if Path(path) == source:
|
||||||
|
Path(target).write_bytes(b"partial")
|
||||||
|
raise Interrupt()
|
||||||
|
return original(path, target, *args, **kwargs)
|
||||||
|
|
||||||
|
with patch.object(stage_runtime.shutil, "copy2", interrupt_copy):
|
||||||
|
with self.assertRaises(Interrupt):
|
||||||
|
stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
|
||||||
|
self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
|
||||||
|
with stage_runtime.staging_lock(destination.parent):
|
||||||
|
stage_runtime.recover_staging(destination)
|
||||||
|
self.assertEqual(list(destination.parent.glob(".resources.stage-*")), [])
|
||||||
|
self.assertFalse((destination.parent / ".searchi-stage.json").exists())
|
||||||
|
|
||||||
@unittest.skipUnless(os.name == "nt", "Windows app-local CRT staging")
|
@unittest.skipUnless(os.name == "nt", "Windows app-local CRT staging")
|
||||||
def test_missing_crt_notice_preserves_existing_staged_worker(self):
|
def test_missing_crt_notice_preserves_existing_staged_worker(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
|||||||
Reference in new issue
Block a user