build: package universal Windows runtime and verify automatic CPU fallback
This commit is contained in:
1 parent
0fcc8ec21c
commit
278b09f82c
16 files changed
+695
-34
No files matched your search
@@ -8,6 +8,8 @@ dist/
|
||||
__pycache__/
|
||||
/apps/desktop/src-tauri/gen/
|
||||
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
|
||||
/apps/desktop/src-tauri/resources/vulkan-1.dll
|
||||
/apps/desktop/src-tauri/resources/VULKAN-LICENSE.txt
|
||||
/apps/desktop/src-tauri/resources/embeddinggemma2/
|
||||
/apps/desktop/src-tauri/resources/media-runtime/bin/
|
||||
/apps/desktop/src-tauri/icons/android/
|
||||
|
||||
@@ -142,6 +142,10 @@ How quickly a folder becomes searchable depends on its size and contents.
|
||||
The desktop keeps its search model warm for subsequent searches.
|
||||
Use **Reindex** after adding or changing files in an indexed folder.
|
||||
|
||||
searchi automatically chooses supported graphics acceleration when available
|
||||
and falls back to CPU when it cannot use it. You do not need a graphics
|
||||
development toolkit. Hardware and release validation are still in progress.
|
||||
|
||||
## Private by design
|
||||
|
||||
You choose the folders to index. By default, file contents, previews and search
|
||||
|
||||
@@ -48,6 +48,8 @@ class Encoder {
|
||||
}
|
||||
bool open(bool gpu) {
|
||||
auto mp = llama_model_default_params();
|
||||
static ggml_backend_dev_t cpu_devices[] = {nullptr};
|
||||
if (!gpu) mp.devices = cpu_devices;
|
||||
mp.n_gpu_layers = gpu ? 999 : 0;
|
||||
mp.progress_callback = [](float,void *) {return true;};
|
||||
model = llama_model_load_from_file(model_path.c_str(),mp);
|
||||
@@ -55,6 +57,7 @@ class Encoder {
|
||||
auto cp = llama_context_default_params();
|
||||
cp.n_ctx = context_tokens; cp.n_batch = context_tokens; cp.n_ubatch = context_tokens;
|
||||
cp.n_seq_max = 1; cp.kv_unified = true; cp.embeddings = true;
|
||||
cp.offload_kqv = gpu; cp.op_offload = gpu;
|
||||
cp.pooling_type = LLAMA_POOLING_TYPE_MEAN;
|
||||
cp.n_threads = threads; cp.n_threads_batch = threads;
|
||||
context = llama_init_from_model(model,cp);
|
||||
@@ -327,6 +330,16 @@ int main(int argc, char ** argv) {
|
||||
require(!path.empty() && threads >= 1 && threads <= 8,"Model path required; threads must be 1–8");
|
||||
require(device == "cpu" || device == "auto","Device must be cpu or auto");
|
||||
bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1";
|
||||
// Disable backend registration before llama initializes its registry.
|
||||
// A GPU-capable distribution can therefore be tested on CPU without
|
||||
// probing/enabling the Vulkan provider in the Windows VM.
|
||||
if (force_cpu || device == "cpu") {
|
||||
#ifdef _WIN32
|
||||
require(_putenv_s("GGML_DISABLE_VULKAN", "1") == 0, "Cannot disable Vulkan for CPU startup");
|
||||
#else
|
||||
require(setenv("GGML_DISABLE_VULKAN", "1", 1) == 0, "Cannot disable Vulkan for CPU startup");
|
||||
#endif
|
||||
}
|
||||
Encoder encoder(path,threads,device == "auto" && !force_cpu);
|
||||
std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
|
||||
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true},{"code_query_supported",true}}).dump() << '\n' << std::flush;
|
||||
|
||||
@@ -78,6 +78,27 @@ impl Drop for GemmaEmbedder {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct StartupTransportFailure(String);
|
||||
impl std::fmt::Display for StartupTransportFailure {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for StartupTransportFailure {}
|
||||
|
||||
/// Retry only process/transport failure, never an incompatible protocol or model.
|
||||
fn start_with_cpu_fallback<T>(cpu: bool, mut start: impl FnMut(bool) -> Result<T>) -> Result<T> {
|
||||
match start(cpu) {
|
||||
Err(error) if !cpu && error.downcast_ref::<StartupTransportFailure>().is_some() => {
|
||||
start(true).with_context(|| {
|
||||
format!("CPU initialization failed after automatic worker startup failure: {error}")
|
||||
})
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
impl GemmaEmbedder {
|
||||
/// Load the verified Google model and explicitly supplied local worker.
|
||||
/// Production selects available acceleration with CPU fallback. Tests set
|
||||
@@ -113,7 +134,16 @@ impl GemmaEmbedder {
|
||||
);
|
||||
let force_cpu = std::env::var("SEARCHI_FORCE_CPU").is_ok_and(|value| value == "1");
|
||||
let cpu = force_cpu || matches!(device, Device::Cpu);
|
||||
let mut command = Command::new(&worker);
|
||||
start_with_cpu_fallback(cpu, |cpu| {
|
||||
Self::start_verified(&model_dir, &worker, threads, cpu)
|
||||
})
|
||||
}
|
||||
|
||||
// Both attempts share the already validated model/worker paths. CPU fallback
|
||||
// does not repeat the expensive hash check or weaken the handshake contract.
|
||||
fn start_verified(model_dir: &Path, worker: &Path, threads: usize, cpu: bool) -> Result<Self> {
|
||||
let model = model_dir.join(MODEL_FILE);
|
||||
let mut command = Command::new(worker);
|
||||
command
|
||||
.args(["--model"])
|
||||
.arg(model)
|
||||
@@ -132,9 +162,14 @@ impl GemmaEmbedder {
|
||||
use std::os::windows::process::CommandExt;
|
||||
command.creation_flags(0x08000000); // CREATE_NO_WINDOW
|
||||
}
|
||||
let mut child = command.spawn().context("Cannot start local Gemma worker")?;
|
||||
let stdin = child.stdin.take().context("Worker stdin missing")?;
|
||||
let stdout = child.stdout.take().context("Worker stdout missing")?;
|
||||
let mut child = command.spawn().map_err(|error| {
|
||||
StartupTransportFailure(format!("Cannot start local Gemma worker: {error}"))
|
||||
})?;
|
||||
let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
anyhow::bail!("Gemma worker protocol pipes missing");
|
||||
};
|
||||
let (sender, responses) = mpsc::sync_channel(1);
|
||||
thread::spawn(move || {
|
||||
let mut reader = BufReader::new(stdout);
|
||||
@@ -151,7 +186,16 @@ impl GemmaEmbedder {
|
||||
other => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
anyhow::bail!("Gemma worker did not initialize: {other:?}");
|
||||
// Malformed/oversized JSON is a protocol violation, not evidence
|
||||
// that hardware acceleration failed. Do not mask it with retry.
|
||||
if matches!(&other, Ok(Err(error)) if error.starts_with("Invalid Gemma worker JSON:") || error == "Gemma worker response exceeds protocol byte budget")
|
||||
{
|
||||
anyhow::bail!("Invalid Gemma worker startup protocol: {other:?}");
|
||||
}
|
||||
return Err(StartupTransportFailure(format!(
|
||||
"Gemma worker did not initialize: {other:?}"
|
||||
))
|
||||
.into());
|
||||
}
|
||||
};
|
||||
let info: WorkerInfo = match serde_json::from_value(ready) {
|
||||
@@ -181,8 +225,8 @@ impl GemmaEmbedder {
|
||||
info,
|
||||
healthy: true,
|
||||
force_cpu: cpu,
|
||||
model_dir,
|
||||
worker_path: worker,
|
||||
model_dir: model_dir.to_path_buf(),
|
||||
worker_path: worker.to_path_buf(),
|
||||
projector_verified: false,
|
||||
query_profile: None,
|
||||
})
|
||||
@@ -666,6 +710,73 @@ fn verify_artifact(path: &Path, bytes: u64, sha: &str) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn automatic_startup_transport_failure_retries_cpu_once() {
|
||||
let mut attempts = Vec::new();
|
||||
let result = start_with_cpu_fallback(false, |cpu| {
|
||||
attempts.push(cpu);
|
||||
if cpu {
|
||||
Ok("cpu ready")
|
||||
} else {
|
||||
Err(StartupTransportFailure("worker closed its output before ready".into()).into())
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(result, "cpu ready");
|
||||
assert_eq!(attempts, [false, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_failure_never_loops_or_retries_explicit_cpu() {
|
||||
for cpu in [false, true] {
|
||||
let mut attempts = Vec::new();
|
||||
let result: Result<()> = start_with_cpu_fallback(cpu, |selected_cpu| {
|
||||
attempts.push(selected_cpu);
|
||||
Err(StartupTransportFailure("worker exited".into()).into())
|
||||
});
|
||||
let error = format!("{:#}", result.unwrap_err());
|
||||
assert_eq!(attempts, if cpu { vec![true] } else { vec![false, true] });
|
||||
assert!(error.contains("worker exited"));
|
||||
if !cpu {
|
||||
assert!(error.contains("CPU initialization failed"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_startup_protocol_is_not_retried() {
|
||||
let mut attempts = 0;
|
||||
let result: Result<()> = start_with_cpu_fallback(false, |_| {
|
||||
attempts += 1;
|
||||
let mut response = BufReader::new(&b"not JSON\n"[..]);
|
||||
let error = read_response(&mut response).unwrap_err();
|
||||
assert!(error.starts_with("Invalid Gemma worker JSON:"));
|
||||
anyhow::bail!("Invalid Gemma worker startup protocol: {error}");
|
||||
});
|
||||
assert!(result.unwrap_err().to_string().contains("startup protocol"));
|
||||
assert_eq!(attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incompatible_startup_contract_is_not_retried() {
|
||||
let mut attempts = 0;
|
||||
let result: Result<()> = start_with_cpu_fallback(false, |_| {
|
||||
attempts += 1;
|
||||
let handshake = json!({"ready":true,"model":"other-model","dimensions":768,"device":"cpu","threads":1,"runtime":"other-runtime"});
|
||||
let info: WorkerInfo = serde_json::from_value(handshake)?;
|
||||
ensure!(
|
||||
info.model == "google/embeddinggemma-2",
|
||||
"Gemma worker handshake violates configured model/device contract"
|
||||
);
|
||||
Ok(())
|
||||
});
|
||||
assert!(result
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("model/device contract"));
|
||||
assert_eq!(attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_text_worker_handshake_does_not_claim_media_support() {
|
||||
let mut handshake = json!({"ready":true,"model":"google/embeddinggemma-2","dimensions":768,"device":"cpu","threads":1,"runtime":"b11461-no-logits-v1"});
|
||||
|
||||
+8
-3
@@ -89,8 +89,11 @@ search starts a model process; the desktop retains a prewarmed query worker.
|
||||
Normal distribution worker builds and staging default to `auto`: Metal on
|
||||
macOS, Vulkan on Windows/Linux, with CPU fallback in the same application.
|
||||
Users do not choose a CPU/GPU edition or rebuild for their graphics card.
|
||||
Windows/Linux build machines need the Vulkan SDK including `glslc`; absence
|
||||
fails release preparation instead of quietly generating a CPU-only runtime.
|
||||
Windows/Linux build machines need the Vulkan SDK including `glslc`. Windows
|
||||
x64 provisioning now downloads the pinned, checksum-verified official SDK into
|
||||
the workspace using copy-only mode; it does not install drivers or change the
|
||||
system environment. Linux builders provision their distro's Vulkan development
|
||||
tools. Missing required tools fail release preparation rather than omitting GPU.
|
||||
The compilation SDK is not an end-user requirement. Explicit `--backend cpu`
|
||||
commands in the VM development sections are internal test configurations.
|
||||
|
||||
@@ -133,7 +136,9 @@ matching-architecture Google worker, FFmpeg/FFprobe and notices. It excludes
|
||||
model caches, local settings and indexes. `PACKAGE.json` records SHA-256 for
|
||||
each included file. Extract the entire archive before launching `searchi.exe`.
|
||||
First launch uses normal verified model setup. WebView2 and the applicable
|
||||
Microsoft C++ runtime remain Windows prerequisites.
|
||||
Microsoft C++ runtime remain Windows prerequisites. The Windows archive includes
|
||||
the verified Vulkan loader DLL and its license; GPU availability is detected at
|
||||
runtime, with CPU fallback. The SDK is used only to compile the app.
|
||||
|
||||
This is an unsigned development archive, not a production installer. Native
|
||||
launch, clean-machine dependencies, upgrades and signatures must still be
|
||||
|
||||
@@ -53,6 +53,12 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
fallback; missing Vulkan/glslc fails configuration rather than hiding GPU loss.
|
||||
- [ ] Provision release-build GPU SDKs and verify one hardware-detecting package
|
||||
per OS/architecture, including missing-driver fallback and dependency closure.
|
||||
- [x] Windows-x64 official pinned SDK copied into workspace with verified cache
|
||||
provenance; universal CPU/Vulkan worker and embedded-frontend desktop built.
|
||||
- [x] Real Windows archive integrity, packaged decoder startup and auto-mode
|
||||
CPU inference with a controlled no-GPU-provider registry; Google CPU parity.
|
||||
- [ ] Actual missing/incompatible GPU drivers, precision, device loss and memory
|
||||
pressure on release hardware; native GUI and clean-machine archive launch.
|
||||
- [x] Windows desktop archive packager includes explicit runtime/helper files
|
||||
and notices, checks architecture and records per-file checksums; excludes
|
||||
model caches and user data. Controlled packaging regression tests pass.
|
||||
|
||||
@@ -40,6 +40,13 @@ must have the relevant compilation SDK. Missing Vulkan/glslc now fails the
|
||||
normal build rather than silently omitting acceleration. Actual GPU parity,
|
||||
loader/dependency packaging and no-driver CPU fallback still require evidence.
|
||||
|
||||
Windows x64 build scripts provision the pinned official Vulkan SDK locally,
|
||||
verify its provenance and stage the app-local loader with its notice. Forced
|
||||
CPU startup disables Vulkan registration before the backend registry initializes;
|
||||
CPU contexts disable device-operation offload. Automatic startup transport
|
||||
failure now restarts once on CPU without weakening model/protocol validation.
|
||||
GPU inference failure also retains the bounded existing CPU retry.
|
||||
|
||||
| Artifact | Bytes | SHA256 |
|
||||
| --- | ---: | --- |
|
||||
| embeddinggemma-2-Q8_0.gguf | 309,855,456 | 2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135 |
|
||||
|
||||
+28
-13
@@ -7,19 +7,34 @@ and `SEARCHI_FORCE_CPU=1`.
|
||||
|
||||
## Current integration checkpoint (2026-10-12)
|
||||
|
||||
The Windows archive packager has two controlled Python regressions: complete
|
||||
required file collection with checksum verification and user-settings exclusion,
|
||||
and mismatched worker architecture preserving the previous archive. Both pass.
|
||||
An actual CPU-only Tauri embedded-frontend Release build has been started;
|
||||
the embedded-frontend Release application build completed in 7m 25s. CPU-only
|
||||
describes its VM test context, not a supported end-user edition. The normal
|
||||
worker build/staging defaults are now auto and require GPU compilation support
|
||||
alongside CPU fallback. The previously staged CPU worker remains an internal
|
||||
test artifact; archive creation, automatic-runtime dependency packaging, native
|
||||
launch outside the source tree and clean-machine installation remain unverified.
|
||||
Actual `auto` configuration correctly rejected this VM's missing Vulkan library,
|
||||
headers and `glslc`; no GPU-capable worker/package is claimed. Both staging
|
||||
regressions still pass using explicit internal CPU test configuration.
|
||||
The Windows-x64 universal worker built with CPU and Vulkan enabled, using the
|
||||
pinned official LunarG 1.4.363.0 SDK (download hash and publisher signature
|
||||
verified). Copy-only installation and later provenance/swap validation ran in
|
||||
the workspace; no driver/registry installation or GPU inference occurred.
|
||||
The desktop embedded-frontend Release build completed and was rebuilt with
|
||||
startup fallback. The generated unsigned Windows archive includes its worker,
|
||||
app-local Vulkan loader, FFmpeg/FFprobe and named notices. Six packaging
|
||||
regressions verify exact archived-byte hashes, changes during copying, notices,
|
||||
architecture mismatch, failed-write cleanup and preservation of previous output.
|
||||
|
||||
The actual extracted package's worker was requested in `auto` mode, with
|
||||
`SEARCHI_FORCE_CPU` absent and Vulkan disabled only in that child. It detected
|
||||
a CPU-only backend registry and produced a real normalized 768d code vector;
|
||||
its decoder started successfully. All 13 recorded package files passed hash
|
||||
checks. [Raw package evidence](evidence/windows-package-cpu-fallback.json).
|
||||
The final local archive is 140,971,051 bytes; its digest is included in that
|
||||
evidence. It has not been published as a production release.
|
||||
This is controlled no-provider fallback, not a physical GPU/driver test or
|
||||
native GUI/clean-machine certification. Actual GPU precision/parity, native
|
||||
application interaction, signed installers and other OS/architecture gates stay open.
|
||||
Two real code tests and existing Google text-reference/tail parity passed on
|
||||
the GPU-capable worker's forced-CPU path. Four startup regressions check one
|
||||
CPU retry after automatic startup transport failure while preserving invalid
|
||||
protocol/model rejection and explicit-CPU behavior. Eight Vulkan-provisioning
|
||||
tests verify cache provenance, bounds, atomic loader reuse and SDK swap rollback.
|
||||
The integrated workspace and native desktop tests passed, along with strict
|
||||
all-target Rust Clippy, frontend Release build and formatting checks. Gitea
|
||||
Actions remain disabled at Karl's request.
|
||||
|
||||
Offline staging now verifies the native license and every bundled Google asset
|
||||
before modifying resources. Two Python regressions verify missing-model rejection
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"architecture": "x86_64",
|
||||
"verified_files": 13,
|
||||
"archive_sha256": "7fa9ccc87ee4fdd0d539898853c47e5b125b42518337a4a2d8393e98c54fb8b8",
|
||||
"worker_requested_device": "auto",
|
||||
"worker_actual_device": "cpu",
|
||||
"no_gpu_condition": "GGML_DISABLE_VULKAN=1 only in child; SEARCHI_FORCE_CPU absent",
|
||||
"worker_startup_query_and_exit_ms": 2509.7244000062346,
|
||||
"query_dimensions": 768,
|
||||
"decoder": "ffprobe version n9.0.2-27-g7e93014c5b-20261012 Copyright (c) 2007-2026 the FFmpeg developers",
|
||||
"scope": "Real packaged worker default-mode CPU fallback with disabled provider; no GPU, native UI or clean-machine certification"
|
||||
}
|
||||
@@ -22,6 +22,11 @@ def main():
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.jobs <= 16:
|
||||
parser.error("jobs must be 1–16")
|
||||
vulkan_loader = None
|
||||
if os.name == "nt" and args.backend in {"auto", "vulkan"}:
|
||||
from provision_vulkan import provision
|
||||
sdk, vulkan_loader = provision()
|
||||
os.environ["VULKAN_SDK"] = str(sdk)
|
||||
source = ROOT / ".searchi/references/llama.cpp"
|
||||
build = ROOT / ".searchi/runtime" / args.backend
|
||||
if not source.exists():
|
||||
@@ -51,6 +56,9 @@ def main():
|
||||
"-DCMAKE_BUILD_TYPE=Release", f"-DLLAMA_CPP_DIR={source}",
|
||||
f"-DSEARCHI_ACCELERATION={args.backend}"])
|
||||
run([cmake_exe, "--build", build, "--config", "Release", "--parallel", args.jobs])
|
||||
if vulkan_loader:
|
||||
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
|
||||
shutil.copy2(vulkan_loader / name, build / "Release" / name)
|
||||
print(f"Runtime build complete: {build}")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Verify a real Windows package and its default-mode no-GPU CPU inference.
|
||||
|
||||
No desktop UI or GPU inference is exercised. The upstream Vulkan provider is
|
||||
disabled only inside the child process to expose a genuinely CPU-only registry.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--archive", type=Path, default=ROOT / ".searchi/packages/searchi-windows.zip")
|
||||
parser.add_argument("--models", type=Path, default=ROOT / ".searchi/models/embeddinggemma2")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / ".searchi/packages/smoke.json")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||
model = args.models / manifest["files"][0]["path"]
|
||||
with model.open("rb") as stream:
|
||||
assert hashlib.file_digest(stream, "sha256").hexdigest() == manifest["files"][0]["sha256"]
|
||||
parent = ROOT / ".searchi/packages"
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="smoke-", dir=parent) as folder:
|
||||
target = Path(folder).resolve()
|
||||
assert target.is_relative_to(parent.resolve())
|
||||
with zipfile.ZipFile(args.archive) as archive:
|
||||
package = json.loads(archive.read("PACKAGE.json"))
|
||||
for record in package["files"]:
|
||||
path = target / record["path"]
|
||||
assert path.resolve().is_relative_to(target)
|
||||
data = archive.read(record["path"])
|
||||
assert len(data) == record["bytes"]
|
||||
assert hashlib.sha256(data).hexdigest() == record["sha256"]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
environment = dict(os.environ, GGML_DISABLE_VULKAN="1")
|
||||
environment.pop("SEARCHI_FORCE_CPU", None)
|
||||
started = time.perf_counter()
|
||||
worker = subprocess.Popen([str(target / "searchi-gemma-worker.exe"), "--model", str(model),
|
||||
"--threads", "1", "--device", "auto"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, text=True, encoding="utf-8", env=environment,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
request = json.dumps({"op": "code_query", "text": "retry failed requests with exponential backoff"}) + "\n"
|
||||
try:
|
||||
stdout, stderr = worker.communicate(request, timeout=120)
|
||||
except BaseException:
|
||||
worker.kill()
|
||||
worker.wait(timeout=10)
|
||||
raise
|
||||
assert worker.returncode == 0, stderr
|
||||
worker_duration_ms = (time.perf_counter()-started)*1000
|
||||
hello, result = [json.loads(line) for line in stdout.splitlines()]
|
||||
assert hello["ready"] and hello["device"] == "cpu" and result["device"] == "cpu" and result["ok"]
|
||||
vector = result["vectors"][0]
|
||||
assert len(vector) == 768 and all(math.isfinite(x) for x in vector)
|
||||
assert abs(sum(x*x for x in vector) - 1) < 0.0001
|
||||
decoder = subprocess.run([str(target / "media-runtime/bin/ffprobe.exe"), "-version"],
|
||||
env=dict(os.environ, SEARCHI_FORCE_CPU="1"), capture_output=True, text=True,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW, check=True, timeout=30)
|
||||
with args.archive.open("rb") as stream:
|
||||
archive_checksum = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
report = {"architecture": package["architecture"], "verified_files": len(package["files"]),
|
||||
"archive_sha256": archive_checksum,
|
||||
"worker_requested_device": "auto", "worker_actual_device": hello["device"],
|
||||
"no_gpu_condition": "GGML_DISABLE_VULKAN=1 only in child; SEARCHI_FORCE_CPU absent",
|
||||
"worker_startup_query_and_exit_ms": worker_duration_ms,
|
||||
"query_dimensions": len(vector), "decoder": decoder.stdout.splitlines()[0],
|
||||
"scope": "Real packaged worker default-mode CPU fallback with disabled provider; no GPU, native UI or clean-machine certification"}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2)+"\n")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Pinned Windows-x64 Vulkan build tools and app-local loader, no GPU execution.
|
||||
|
||||
SDK copy-only mode does not install drivers or change system environment/registry.
|
||||
Checksums are from https://vulkan.lunarg.com/sdk/home/ (2026-10-12).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VERSION = "1.4.363.0"
|
||||
SDK_SHA = "94a82d378f7a5e3e54c9db7d2fb7016af136e14ac0a18dbf0f2f67a36352d141"
|
||||
LOADER_SHA = "a25a927aa8b9f0371048f1861cf88ac3b9bc9b1fb332c42d897c8ab32695769a"
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def download(path, remote, expected, limit):
|
||||
if path.is_file() and digest(path) == expected:
|
||||
return
|
||||
temporary = path.with_suffix(path.suffix + ".partial")
|
||||
url = f"https://sdk.lunarg.com/sdk/download/{VERSION}/windows/{remote}?Human=true"
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "searchi-build-tooling/1.0"})
|
||||
with urllib.request.urlopen(request, timeout=120) as response, temporary.open("wb") as output:
|
||||
count = 0
|
||||
while block := response.read(1024 * 1024):
|
||||
count += len(block)
|
||||
if count > limit:
|
||||
raise ValueError("Vulkan download exceeds size ceiling")
|
||||
output.write(block)
|
||||
if digest(temporary) != expected:
|
||||
raise ValueError("Vulkan download checksum mismatch")
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def sdk_files(sdk):
|
||||
required = [sdk / "Bin/glslc.exe", sdk / "Lib/vulkan-1.lib",
|
||||
sdk / "Include/vulkan/vulkan.h", sdk / "Include/vulkan/vulkan_core.h"]
|
||||
if not all(path.is_file() for path in required):
|
||||
raise RuntimeError("SDK copy did not provide required tools and headers")
|
||||
files = set(required)
|
||||
for directory in [sdk / "Include/vulkan", sdk / "Include/vk_video"]:
|
||||
files.update(path for path in directory.rglob("*") if path.is_file())
|
||||
files.update((sdk / "Bin").glob("*.dll"))
|
||||
if len(files) > 4096:
|
||||
raise ValueError("Vulkan SDK manifest exceeds file count ceiling")
|
||||
return {path.relative_to(sdk).as_posix(): digest(path) for path in sorted(files)}
|
||||
|
||||
|
||||
def sdk_verified(sdk):
|
||||
manifest = sdk / "searchi-provenance.json"
|
||||
try:
|
||||
if manifest.stat().st_size > 1024 * 1024:
|
||||
return False
|
||||
value = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
return (value.get("version") == VERSION and value.get("installer_sha256") == SDK_SHA
|
||||
and value.get("files") == sdk_files(sdk))
|
||||
except (OSError, ValueError, RuntimeError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def write_sdk_provenance(sdk):
|
||||
value = {"version": VERSION, "installer_sha256": SDK_SHA, "files": sdk_files(sdk)}
|
||||
temporary = sdk / "searchi-provenance.json.partial"
|
||||
try:
|
||||
temporary.write_text(json.dumps(value, sort_keys=True), encoding="utf-8")
|
||||
os.replace(temporary, sdk / "searchi-provenance.json")
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def owned_path(directory, path):
|
||||
"""Resolve and reject escaped/reparse targets before directory moves."""
|
||||
owner = directory.resolve()
|
||||
target = path.resolve()
|
||||
if target == owner or not target.is_relative_to(owner):
|
||||
raise ValueError("Vulkan SDK path escapes owned toolchains directory")
|
||||
return target
|
||||
|
||||
|
||||
def install_sdk(directory, sdk, installer):
|
||||
# Qt's copy-only installer refuses an existing root. Use a fresh sibling,
|
||||
# validate it fully, then swap; preserve previous/failed trees for recovery.
|
||||
sdk = owned_path(directory, sdk)
|
||||
staging = owned_path(directory, directory / f"vulkan-sdk-{VERSION}-staging-{uuid.uuid4().hex}")
|
||||
backup = owned_path(directory, directory / f"vulkan-sdk-{VERSION}-backup-{uuid.uuid4().hex}")
|
||||
if staging.exists() or backup.exists():
|
||||
raise RuntimeError("Generated Vulkan staging or backup directory already exists")
|
||||
environment = dict(os.environ, QT_OPENGL="software", SEARCHI_FORCE_CPU="1")
|
||||
subprocess.run([str(installer), "--root", str(staging), "--accept-licenses", "--default-answer",
|
||||
"--confirm-command", "install", "copy_only=1"], env=environment, check=True,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
write_sdk_provenance(staging)
|
||||
if not sdk_verified(staging):
|
||||
raise RuntimeError("Fresh Vulkan SDK failed provenance validation")
|
||||
# Re-resolve immediately before mutation; no recursive removal is performed.
|
||||
owned_path(directory, staging)
|
||||
owned_path(directory, sdk)
|
||||
owned_path(directory, backup)
|
||||
previous = sdk.exists()
|
||||
if previous:
|
||||
os.replace(sdk, backup)
|
||||
try:
|
||||
os.replace(staging, sdk)
|
||||
except OSError:
|
||||
if previous and not sdk.exists():
|
||||
owned_path(directory, backup)
|
||||
owned_path(directory, sdk)
|
||||
os.replace(backup, sdk)
|
||||
raise
|
||||
|
||||
|
||||
def stage_loader(archive, loader):
|
||||
if digest(archive) != LOADER_SHA:
|
||||
raise ValueError("Vulkan runtime archive checksum mismatch")
|
||||
prefix = f"VulkanRT-X64-{VERSION}-Components/"
|
||||
members = []
|
||||
with zipfile.ZipFile(archive) as package:
|
||||
for source, name in [("x64/vulkan-1.dll", "vulkan-1.dll"),
|
||||
("VulkanRT-License.txt", "VULKAN-LICENSE.txt")]:
|
||||
member = package.getinfo(prefix + source)
|
||||
if member.file_size > 16 * 1024 * 1024:
|
||||
raise ValueError("Vulkan runtime member exceeds bound")
|
||||
with package.open(member) as stream:
|
||||
data = stream.read(16 * 1024 * 1024 + 1)
|
||||
if len(data) != member.file_size or len(data) > 16 * 1024 * 1024:
|
||||
raise ValueError("Vulkan runtime member exceeds bound or is truncated")
|
||||
members.append((name, data))
|
||||
loader.mkdir(exist_ok=True)
|
||||
for name, data in members:
|
||||
destination = loader / name
|
||||
if destination.is_file() and digest(destination) == hashlib.sha256(data).hexdigest():
|
||||
continue # Never rewrite a verified DLL mapped by another process.
|
||||
temporary = destination.with_suffix(destination.suffix + ".partial")
|
||||
try:
|
||||
temporary.write_bytes(data)
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def provision():
|
||||
if os.name != "nt" or platform.machine().lower() not in {"amd64", "x86_64"}:
|
||||
raise RuntimeError("Automatic Vulkan tooling provisioning is verified for Windows x64 only")
|
||||
directory = ROOT / ".searchi/toolchains"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
sdk = directory / f"vulkan-sdk-{VERSION}"
|
||||
if not sdk_verified(sdk):
|
||||
installer = directory / f"vulkansdk-windows-X64-{VERSION}.exe"
|
||||
download(installer, installer.name, SDK_SHA, 400 * 1024 * 1024)
|
||||
install_sdk(directory, sdk, installer)
|
||||
archive = directory / f"VulkanRT-X64-{VERSION}-Components.zip"
|
||||
download(archive, "vulkan-runtime-components.zip", LOADER_SHA, 50 * 1024 * 1024)
|
||||
loader = directory / "vulkan-loader"
|
||||
stage_loader(archive, loader)
|
||||
return sdk, loader
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sdk, loader = provision()
|
||||
print(f"Verified SDK: {sdk}\nApp-local loader: {loader}")
|
||||
@@ -42,17 +42,24 @@ def main():
|
||||
if args.bundle_model:
|
||||
manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||
artifacts = verify_bundle(ROOT, manifest)
|
||||
loader = None
|
||||
if os.name == "nt" and args.backend in {"auto", "vulkan"}:
|
||||
from provision_vulkan import provision
|
||||
_, loader = provision()
|
||||
destination = ROOT / "apps/desktop/src-tauri/resources"
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, destination / name)
|
||||
# Include the upstream runtime license with the statically linked helper.
|
||||
shutil.copy2(license_path, destination / "LLAMA-LICENSE.txt")
|
||||
if loader:
|
||||
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
|
||||
shutil.copy2(loader / name, destination / name)
|
||||
if args.bundle_model:
|
||||
model_destination = destination / "embeddinggemma2"
|
||||
model_destination.mkdir(exist_ok=True)
|
||||
for artifact in artifacts:
|
||||
shutil.copy2(artifact, model_destination / artifact.name)
|
||||
print(f"Staged native worker: {destination / name}")
|
||||
print(f"Staged native worker: {destination / source.name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import hashlib
|
||||
import io
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import subprocess
|
||||
import unittest
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
import provision_vulkan as provisioning
|
||||
from provision_vulkan import download, sdk_verified, write_sdk_provenance, stage_loader
|
||||
|
||||
|
||||
class DownloadTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def fake_sdk(sdk):
|
||||
for name in ["Bin/glslc.exe", "Lib/vulkan-1.lib", "Include/vulkan/vulkan.h",
|
||||
"Include/vulkan/vulkan_core.h"]:
|
||||
path = sdk / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(name.encode())
|
||||
|
||||
def test_fresh_install_preserves_previous_cache_and_uses_copy_only(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
sdk = root / "sdk"
|
||||
sdk.mkdir()
|
||||
(sdk / "old-marker").write_bytes(b"previous tree")
|
||||
|
||||
def install(command, **options):
|
||||
fresh = Path(command[command.index("--root") + 1])
|
||||
self.assertFalse(fresh.exists())
|
||||
self.assertIn("copy_only=1", command)
|
||||
self.assertEqual(options["env"]["QT_OPENGL"], "software")
|
||||
self.assertEqual(options["env"]["SEARCHI_FORCE_CPU"], "1")
|
||||
self.fake_sdk(fresh)
|
||||
|
||||
with patch("provision_vulkan.subprocess.run", side_effect=install):
|
||||
provisioning.install_sdk(root, sdk, root / "installer.exe")
|
||||
self.assertTrue(sdk_verified(sdk))
|
||||
backups = list(root.glob("*-backup-*"))
|
||||
self.assertEqual(len(backups), 1)
|
||||
self.assertEqual((backups[0] / "old-marker").read_bytes(), b"previous tree")
|
||||
|
||||
def test_failed_installer_and_failed_swap_preserve_previous_provenance(self):
|
||||
for failure in ["installer", "swap"]:
|
||||
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
sdk = root / "sdk"
|
||||
self.fake_sdk(sdk)
|
||||
write_sdk_provenance(sdk)
|
||||
old_manifest = (sdk / "searchi-provenance.json").read_bytes()
|
||||
|
||||
def install(command, **options):
|
||||
if failure == "installer":
|
||||
raise subprocess.CalledProcessError(1, command)
|
||||
self.fake_sdk(Path(command[command.index("--root") + 1]))
|
||||
|
||||
real_replace = provisioning.os.replace
|
||||
|
||||
def replace(source, destination):
|
||||
if failure == "swap" and "-staging-" in Path(source).name and Path(destination) == sdk:
|
||||
raise PermissionError("injected swap failure")
|
||||
real_replace(source, destination)
|
||||
|
||||
with patch("provision_vulkan.subprocess.run", side_effect=install), \
|
||||
patch("provision_vulkan.os.replace", side_effect=replace):
|
||||
with self.assertRaises((OSError, subprocess.CalledProcessError)):
|
||||
provisioning.install_sdk(root, sdk, root / "installer.exe")
|
||||
self.assertEqual((sdk / "searchi-provenance.json").read_bytes(), old_manifest)
|
||||
self.assertTrue(sdk_verified(sdk))
|
||||
|
||||
def test_install_rejects_target_outside_owned_directory(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
owned = root / "owned"
|
||||
owned.mkdir()
|
||||
with patch("provision_vulkan.subprocess.run", side_effect=AssertionError("installer ran")):
|
||||
with self.assertRaisesRegex(ValueError, "escapes owned"):
|
||||
provisioning.install_sdk(owned, root / "outside", root / "installer.exe")
|
||||
|
||||
def test_sdk_requires_provenance_and_detects_header_corruption(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
sdk = Path(directory)
|
||||
for name in ["Bin/glslc.exe", "Lib/vulkan-1.lib", "Include/vulkan/vulkan.h",
|
||||
"Include/vulkan/vulkan_core.h", "Include/vk_video/video.h"]:
|
||||
path = sdk / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(name.encode())
|
||||
self.assertFalse(sdk_verified(sdk))
|
||||
write_sdk_provenance(sdk)
|
||||
self.assertTrue(sdk_verified(sdk))
|
||||
(sdk / "Include/vk_video/video.h").write_bytes(b"corrupt")
|
||||
self.assertFalse(sdk_verified(sdk))
|
||||
|
||||
def test_verified_loader_is_not_replaced_and_bad_archive_preserves_it(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
archive = root / "runtime.zip"
|
||||
prefix = f"VulkanRT-X64-{provisioning.VERSION}-Components/"
|
||||
with zipfile.ZipFile(archive, "w") as package:
|
||||
package.writestr(prefix + "x64/vulkan-1.dll", b"verified loader")
|
||||
package.writestr(prefix + "VulkanRT-License.txt", b"license")
|
||||
package.writestr("../outside", b"ignored")
|
||||
checksum = provisioning.digest(archive)
|
||||
loader = root / "loader"
|
||||
with patch.object(provisioning, "LOADER_SHA", checksum):
|
||||
stage_loader(archive, loader)
|
||||
with patch("provision_vulkan.os.replace", side_effect=AssertionError("rewrote loader")):
|
||||
stage_loader(archive, loader)
|
||||
archive.write_bytes(b"corrupt")
|
||||
with self.assertRaisesRegex(ValueError, "archive checksum"):
|
||||
stage_loader(archive, loader)
|
||||
self.assertEqual((loader / "vulkan-1.dll").read_bytes(), b"verified loader")
|
||||
self.assertFalse((root / "outside").exists())
|
||||
|
||||
def test_unsupported_architecture_fails_before_filesystem_or_network(self):
|
||||
with patch("provision_vulkan.platform.machine", return_value="ARM64"), \
|
||||
patch("provision_vulkan.Path.mkdir", side_effect=AssertionError("filesystem")), \
|
||||
patch("provision_vulkan.urllib.request.urlopen", side_effect=AssertionError("network")):
|
||||
with self.assertRaisesRegex(RuntimeError, "Windows x64 only"):
|
||||
provisioning.provision()
|
||||
|
||||
def test_verified_cache_needs_no_network(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "sdk.exe"
|
||||
path.write_bytes(b"verified SDK")
|
||||
checksum = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
with patch("provision_vulkan.urllib.request.urlopen", side_effect=AssertionError("unexpected network")):
|
||||
download(path, "sdk.exe", checksum, 100)
|
||||
|
||||
def test_corrupt_or_oversized_download_preserves_previous_file(self):
|
||||
for payload, limit, reason in [(b"wrong", 100, "checksum"), (b"too large", 2, "size ceiling")]:
|
||||
with self.subTest(reason=reason), tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "sdk.exe"
|
||||
path.write_bytes(b"previous cached file")
|
||||
with patch("provision_vulkan.urllib.request.urlopen", return_value=io.BytesIO(payload)):
|
||||
with self.assertRaisesRegex(ValueError, reason):
|
||||
download(path, "sdk.exe", hashlib.sha256(b"expected").hexdigest(), limit)
|
||||
self.assertEqual(path.read_bytes(), b"previous cached file")
|
||||
self.assertFalse(path.with_suffix(".exe.partial").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -33,14 +33,17 @@ def pe_machine(path):
|
||||
def package(root, executable, output):
|
||||
resources = root / "apps/desktop/src-tauri/resources"
|
||||
selected = [(executable, "searchi.exe"), (root / "LICENSE", "LICENSE.txt")]
|
||||
for name in ["searchi-gemma-worker.exe", "LLAMA-LICENSE.txt", "media-runtime/manifest.json",
|
||||
"media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe"]:
|
||||
for name in ["searchi-gemma-worker.exe", "LLAMA-LICENSE.txt", "vulkan-1.dll", "VULKAN-LICENSE.txt", "media-runtime/manifest.json",
|
||||
"media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe",
|
||||
"licenses/Inter-OFL.txt", "licenses/Fluent-MIT.txt", "media-runtime/licenses/LICENSE.txt"]:
|
||||
selected.append((resources / name, name))
|
||||
for directory in ["licenses", "media-runtime/licenses"]:
|
||||
notices = list((resources / directory).rglob("*.txt"))
|
||||
if not notices:
|
||||
raise ValueError(f"Required notices are missing: {directory}")
|
||||
selected.extend((path, path.relative_to(resources).as_posix()) for path in notices)
|
||||
selected_names = {name for _, name in selected}
|
||||
selected.extend((path, path.relative_to(resources).as_posix()) for path in notices
|
||||
if path.relative_to(resources).as_posix() not in selected_names)
|
||||
machine = pe_machine(executable)
|
||||
architecture = {0x8664: "x86_64", 0xAA64: "aarch64"}.get(machine)
|
||||
if architecture is None:
|
||||
@@ -50,7 +53,7 @@ def package(root, executable, output):
|
||||
raise ValueError(f"Required package file is missing: {name}")
|
||||
if path != executable and path != root / "LICENSE" and not path.resolve().is_relative_to(resources.resolve()):
|
||||
raise ValueError("Resource resolves outside the staging directory")
|
||||
if name.endswith(".exe") and pe_machine(path) != machine:
|
||||
if name.endswith((".exe", ".dll")) and pe_machine(path) != machine:
|
||||
raise ValueError(f"Executable architecture differs: {name}")
|
||||
records = []
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -59,10 +62,17 @@ def package(root, executable, output):
|
||||
try:
|
||||
with zipfile.ZipFile(temporary_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
||||
for path, name in selected:
|
||||
with path.open("rb") as stream:
|
||||
checksum = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
archive.write(path, name)
|
||||
records.append({"path": name, "bytes": path.stat().st_size, "sha256": checksum})
|
||||
# Hash and count the same blocks written to the archive: a
|
||||
# second source read or subsequent stat can describe different
|
||||
# bytes when a staged file changes during release preparation.
|
||||
checksum = hashlib.sha256()
|
||||
count = 0
|
||||
with path.open("rb") as stream, archive.open(name, "w", force_zip64=True) as entry:
|
||||
while block := stream.read(1024 * 1024):
|
||||
entry.write(block)
|
||||
checksum.update(block)
|
||||
count += len(block)
|
||||
records.append({"path": name, "bytes": count, "sha256": checksum.hexdigest()})
|
||||
archive.writestr("PACKAGE.json", json.dumps({"platform": "windows", "architecture": architecture,
|
||||
"status": "unsigned development package; clean-machine verification pending", "files": records}, indent=2))
|
||||
archive.writestr("START-HERE.txt", "searchi — Find what matters. Locally.\n\n"
|
||||
|
||||
@@ -4,6 +4,7 @@ from pathlib import Path
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import zipfile
|
||||
|
||||
from package_windows import package
|
||||
@@ -26,9 +27,9 @@ class PackageTests(unittest.TestCase):
|
||||
app = root / "build/app.exe"
|
||||
executable(app)
|
||||
(root / "LICENSE").write_text("application notice")
|
||||
for name in ["searchi-gemma-worker.exe", "media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe"]:
|
||||
for name in ["searchi-gemma-worker.exe", "vulkan-1.dll", "media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe"]:
|
||||
executable(resources / name)
|
||||
for name in ["LLAMA-LICENSE.txt", "licenses/font.txt", "media-runtime/licenses/LICENSE.txt", "media-runtime/manifest.json"]:
|
||||
for name in ["LLAMA-LICENSE.txt", "VULKAN-LICENSE.txt", "licenses/Inter-OFL.txt", "licenses/Fluent-MIT.txt", "media-runtime/licenses/LICENSE.txt", "media-runtime/manifest.json"]:
|
||||
path = resources / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("notice")
|
||||
@@ -44,6 +45,8 @@ class PackageTests(unittest.TestCase):
|
||||
self.assertIn("media-runtime/bin/ffprobe.exe", archive.namelist())
|
||||
for record in json.loads(archive.read("PACKAGE.json"))["files"]:
|
||||
self.assertEqual(hashlib.sha256(archive.read(record["path"])).hexdigest(), record["sha256"])
|
||||
self.assertEqual(len(archive.read(record["path"])), record["bytes"])
|
||||
self.assertEqual(len(archive.namelist()), len(set(archive.namelist())))
|
||||
|
||||
def test_mixed_architecture_preserves_previous_archive(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
@@ -54,6 +57,62 @@ class PackageTests(unittest.TestCase):
|
||||
package(root, app, output)
|
||||
self.assertEqual(output.read_bytes(), b"previous package")
|
||||
|
||||
def test_changed_source_records_exact_archived_bytes(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root, _, app, output = self.fixture(directory)
|
||||
original_open = zipfile.ZipFile.open
|
||||
|
||||
def mutate_source(archive, name, mode="r", *args, **kwargs):
|
||||
if name == "searchi.exe" and mode == "w":
|
||||
# Change bytes after preflight and after the input stream
|
||||
# opens, at the point ZIP copying starts.
|
||||
app.write_bytes(app.read_bytes() + b"changed while packaging")
|
||||
return original_open(archive, name, mode, *args, **kwargs)
|
||||
|
||||
with mock.patch.object(zipfile.ZipFile, "open", mutate_source):
|
||||
package(root, app, output)
|
||||
with zipfile.ZipFile(output) as archive:
|
||||
content = archive.read("searchi.exe")
|
||||
record = json.loads(archive.read("PACKAGE.json"))["files"][0]
|
||||
self.assertIn(b"changed while packaging", content)
|
||||
self.assertEqual(record["bytes"], len(content))
|
||||
self.assertEqual(record["sha256"], hashlib.sha256(content).hexdigest())
|
||||
|
||||
def test_mid_write_failure_preserves_previous_archive_and_cleans_temporary(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root, _, app, output = self.fixture(directory)
|
||||
output.write_bytes(b"previous package")
|
||||
original_open = zipfile.ZipFile.open
|
||||
|
||||
def fail_second_entry(archive, name, mode="r", *args, **kwargs):
|
||||
if name == "LICENSE.txt" and mode == "w":
|
||||
raise OSError("simulated storage failure")
|
||||
return original_open(archive, name, mode, *args, **kwargs)
|
||||
|
||||
with mock.patch.object(zipfile.ZipFile, "open", fail_second_entry):
|
||||
with self.assertRaisesRegex(OSError, "storage failure"):
|
||||
package(root, app, output)
|
||||
self.assertEqual(output.read_bytes(), b"previous package")
|
||||
self.assertEqual(list(output.parent.glob("*.partial")), [])
|
||||
|
||||
def test_missing_named_notice_preserves_previous_archive(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root, resources, app, output = self.fixture(directory)
|
||||
output.write_bytes(b"previous package")
|
||||
(resources / "licenses/Inter-OFL.txt").unlink()
|
||||
with self.assertRaisesRegex(ValueError, "Inter-OFL"):
|
||||
package(root, app, output)
|
||||
self.assertEqual(output.read_bytes(), b"previous package")
|
||||
|
||||
def test_mixed_loader_architecture_preserves_previous_archive(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root, resources, app, output = self.fixture(directory)
|
||||
output.write_bytes(b"previous package")
|
||||
executable(resources / "vulkan-1.dll", 0xAA64)
|
||||
with self.assertRaisesRegex(ValueError, "architecture differs: vulkan-1.dll"):
|
||||
package(root, app, output)
|
||||
self.assertEqual(output.read_bytes(), b"previous package")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user