diff --git a/AGENTS.md b/AGENTS.md index 606a91e..a96bfa2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,9 @@ readiness without completed release gates and recorded evidence. permission to index in a query session. Legacy workers need an explicit update, never a silent larger-context fallback. Keep private commitment, resident memory, model verification and cached startup measurements distinct. +- Windows native workers must preserve UTF-16 arguments and explicitly convert + them to UTF-8 for ggml; never depend on the system ANSI code page. Keep genuine + Unicode model and relocated runtime/DLL tests, with owned fixture cleanup. - Treat native filesystem events as bounded hints; retain work across busy scans and verify periodically. Use targeted, cancellable reconciliation for events. Media hints must advance durable change revisions even with identical stats; diff --git a/Cargo.lock b/Cargo.lock index f098e33..369c0aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2300,6 +2300,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tempfile", "tokenizers", ] diff --git a/crates/searchi-semantic/Cargo.toml b/crates/searchi-semantic/Cargo.toml index 49d0dc3..988a724 100644 --- a/crates/searchi-semantic/Cargo.toml +++ b/crates/searchi-semantic/Cargo.toml @@ -21,4 +21,5 @@ serde.workspace = true serde_json.workspace = true [dev-dependencies] +tempfile = "3" serde_json.workspace = true diff --git a/crates/searchi-semantic/native/CMakeLists.txt b/crates/searchi-semantic/native/CMakeLists.txt index 80aa4c9..c2b3b40 100644 --- a/crates/searchi-semantic/native/CMakeLists.txt +++ b/crates/searchi-semantic/native/CMakeLists.txt @@ -67,6 +67,10 @@ set(MTMD_VIDEO OFF CACHE BOOL "Frames decoded by bounded media adapter" FORCE) set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE) add_subdirectory("${LLAMA_CPP_DIR}" llama-build) add_executable(searchi-gemma-worker worker.cpp) +if(MINGW) + # Windows argument conversion uses wmain; keep the same wide CRT entry. + target_link_options(searchi-gemma-worker PRIVATE -municode) +endif() target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor") target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd ${CMAKE_DL_LIBS}) if(SEARCHI_CPU_DISPATCH) diff --git a/crates/searchi-semantic/native/worker.cpp b/crates/searchi-semantic/native/worker.cpp index 3f294f0..1aee71b 100644 --- a/crates/searchi-semantic/native/worker.cpp +++ b/crates/searchi-semantic/native/worker.cpp @@ -15,16 +15,17 @@ #include #include #include -#ifdef SEARCHI_CPU_DISPATCH -#include #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #define NOMINMAX #include -#elif defined(__linux__) +#endif +#ifdef SEARCHI_CPU_DISPATCH +#include +#if defined(__linux__) #include #include -#else +#elif !defined(_WIN32) #error Unsupported platform for packaged CPU dispatch #endif #endif @@ -384,7 +385,7 @@ public: } }; -int main(int argc, char ** argv) { +static int worker_main(int argc, const char * const * argv) { try { std::string path, device = "auto", role = "indexing"; int threads = 1; for (int i = 1; i < argc; ++i) { @@ -447,3 +448,35 @@ int main(int argc, char ** argv) { return 1; } } + +#ifdef _WIN32 +// The narrow CRT argv uses the system ANSI code page (commonly1252), whereas +// ggml_fopen requires UTF-8. Preserve Windows UTF-16 arguments before parsing; +// model/runtime directories may contain any valid Unicode user/folder name. +int wmain(int argc, wchar_t ** argv) { + try { + std::vector utf8; + utf8.reserve(size_t(argc)); + for (int i = 0; i < argc; ++i) { + int bytes = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, argv[i], -1, nullptr, 0, nullptr, nullptr); + require(bytes > 0, "Worker argument contains invalid Windows Unicode"); + std::string argument(size_t(bytes), '\0'); + require(WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, argv[i], -1, argument.data(), bytes, nullptr, nullptr) == bytes, + "Cannot convert worker argument to UTF-8"); + argument.pop_back(); // Conversion includes the terminating NUL. + utf8.push_back(std::move(argument)); + } + std::vector arguments; + arguments.reserve(utf8.size()); + for (const auto & argument : utf8) arguments.push_back(argument.c_str()); + return worker_main(argc, arguments.data()); + } catch (const std::exception & error) { + std::cerr << "Searchi Gemma worker: " << error.what() << '\n'; + return 1; + } +} +#else +int main(int argc, char ** argv) { + return worker_main(argc, argv); +} +#endif diff --git a/crates/searchi-semantic/tests/unicode_windows_cpu.rs b/crates/searchi-semantic/tests/unicode_windows_cpu.rs new file mode 100644 index 0000000..9fe0030 --- /dev/null +++ b/crates/searchi-semantic/tests/unicode_windows_cpu.rs @@ -0,0 +1,75 @@ +//! Real Windows filename/CRT boundary, independent of the active ANSI code page. +#![cfg(windows)] +use anyhow::{Context, Result}; +use searchi_semantic::{Device, GemmaEmbedder}; +use std::path::PathBuf; + +#[test] +#[ignore = "Requires verified Google model and current Windows CPU-capable worker"] +fn unicode_model_directory_preserves_query_and_indexing_roles() -> Result<()> { + let models = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set models")) + .canonicalize()?; + let worker = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker")) + .canonicalize()?; + // TempDir owns only new names. Create on the source volume so the fixture + // can hardlink the verified backbone without copying or changing its bytes. + let fixture = tempfile::Builder::new() + .prefix("searchi-unicode-") + .tempdir_in(models.parent().context("Model directory needs parent")?)?; + let unicode = fixture.path().join("Gemäß_日本_😀"); + std::fs::create_dir(&unicode)?; + let filename = "embeddinggemma-2-Q8_0.gguf"; + std::fs::hard_link(models.join(filename), unicode.join(filename))?; + // Relocate only adjacent runtime files as hardlinks. Windows DLL loading + // and packaged backend discovery must also survive a Unicode install path. + for entry in std::fs::read_dir(worker.parent().context("Worker needs parent")?)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_file() + && (path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("dll")) + || path == worker) + { + std::fs::hard_link(&path, unicode.join(entry.file_name()))?; + } + } + let relocated_worker = unicode.join(worker.file_name().context("Worker needs filename")?); + let mut reference = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?; + let expected = reference.embed_query("cancel the supplier partnership")?; + drop(reference); + let mut query = + GemmaEmbedder::load_query_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?; + assert_eq!(query.info.role, "query"); + assert_eq!(query.info.context_tokens, 512); + assert_eq!(query.info.device, "cpu"); + let actual = query.embed_query("cancel the supplier partnership")?; + let cosine: f64 = actual + .iter() + .zip(&expected) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum(); + assert!( + cosine >= 0.99999, + "Unicode path changed inference: {cosine}" + ); + drop(query); + let mut indexing = + GemmaEmbedder::load_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?; + assert_eq!(indexing.info.role, "indexing"); + assert_eq!(indexing.info.context_tokens, 2048); + assert_eq!( + indexing + .embed_documents( + &["A supplier agreement with sixty days written notice."], + "Supplier" + )? + .len(), + 1 + ); + drop(indexing); + drop(fixture); // Hardlink deletion does not remove the original model. + assert_eq!(std::fs::metadata(models.join(filename))?.len(), 309855456); + println!("Real Unicode model query/indexing roles passed; ASCII/Unicode cosine {cosine:.8}"); + Ok(()) +} diff --git a/docs/WINDOWS-UNICODE.md b/docs/WINDOWS-UNICODE.md new file mode 100644 index 0000000..256772c --- /dev/null +++ b/docs/WINDOWS-UNICODE.md @@ -0,0 +1,62 @@ +# Windows Unicode model paths + +The native worker must preserve filenames from Windows command-line arguments. +Windows narrow CRT `char** argv` uses the active ANSI code page; the pinned +llama/ggml model loader expects UTF-8 filenames. UTF-8 JSON requests alone do +not correct the model path supplied at startup. + +## Observed failure, 2026-10-13 + +An actual CPU query-role startup used a hardlink to the verified Google backbone +under `Gemäß_日本_😀`. The model bytes were unchanged (309,855,456 bytes, +SHA256 `2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135`). +On this Windows VM the active ANSI code page was 1252. The normal worker with +SHA256 `1272797af94c21db1ccabaa3ad996ab57d64f1bf9d33280c61d65c51360deab7` +exited with code 1 before a ready handshake. `gguf_init_from_file` reported an +invalid argument with the Japanese and emoji components replaced by question +marks, and the German component no longer valid UTF-8. The captured raw stderr +is preserved locally in `.searchi/qa/windows-unicode-model-before-stderr.bin`. + +## Implementation + +Windows enters through `wmain`, converts the original UTF-16 arguments to UTF-8 +with `WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS)`, then calls the shared +worker entry. Invalid Unicode fails explicitly. Unix continues to pass its +existing byte arguments through `main`. Normal CPU/GPU selection, protocol, +model validation and role behavior remain unchanged. MinGW receives its required +`-municode` link option, but a MinGW build has not been executed or certified. + +After the fix, normal MSVC worker SHA256 +`c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73` +passed the real Unicode query/indexing regression (21.03 s). A strengthened +regression also relocated the worker and every adjacent DLL as hardlinks under +the Unicode folder: actual query and full document indexing passed (22.45 s), +with ASCII/Unicode query-vector cosine 1.00000001. The temporary fixture names +were verified absent afterward and the original backbone remained intact. +The unchanged query-context/parity/boundary regression passed again (25.83 s). + +A separate native after-probe reported query role, 512-token context, CPU, +`ggml-cpu-cascadelake.dll`, no Vulkan module, and a normalized 768-dimensional +vector. Metadata is retained in `.searchi/qa/windows-unicode-model-after.json`. +The ad-hoc model hardlink was removed only after verifying its absolute path +under the owned QA root and that the directory contained exactly that link. +The original verified model remains present. No GPU was executed. + +The actual regression creates a uniquely owned temporary directory on the +model's volume, hardlinks the backbone under a Unicode child, relocates adjacent +runtime files as hardlinks, and compares +ASCII/Unicode CPU query vectors. It also opens the full indexing role through +that Unicode path and encodes a real document. Temporary-name cleanup leaves +the original verified model intact. No duplicate model download or copy is +needed; hardware acceleration is not executed in this VM. + +```powershell +$env:SEARCHI_FORCE_CPU = '1' +$env:SEARCHI_GEMMA_MODEL_DIR = 'PATH/TO/VERIFIED/MODELS' +$env:SEARCHI_GEMMA_WORKER = 'PATH/TO/CURRENT/searchi-gemma-worker.exe' +cargo test -p searchi-semantic --test unicode_windows_cpu --locked -- --ignored --nocapture +``` + +Native package restaging and installed Unicode-profile paths require their own +checks. This focused model-path regression does not certify every filesystem, +codec, launcher or installer path, or any GPU hardware.