Preserve Unicode Windows arguments when loading native models

AGENTS.md

@@ -58,6 +58,9 @@ readiness without completed release gates and recorded evidence.
   permission to index in a query session. Legacy workers need an explicit update,
   never a silent larger-context fallback. Keep private commitment, resident memory,
   model verification and cached startup measurements distinct.
+- Windows native workers must preserve UTF-16 arguments and explicitly convert
+  them to UTF-8 for ggml; never depend on the system ANSI code page. Keep genuine
+  Unicode model and relocated runtime/DLL tests, with owned fixture cleanup.
 - Treat native filesystem events as bounded hints; retain work across busy scans
   and verify periodically. Use targeted, cancellable reconciliation for events.
   Media hints must advance durable change revisions even with identical stats;

Cargo.lock

@@ -2300,6 +2300,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2 0.10.9",
+ "tempfile",
  "tokenizers",
 ]
 

crates/searchi-semantic/Cargo.toml

@@ -21,4 +21,5 @@ serde.workspace = true
 serde_json.workspace = true
 
 [dev-dependencies]
+tempfile = "3"
 serde_json.workspace = true

crates/searchi-semantic/native/CMakeLists.txt

@@ -67,6 +67,10 @@ set(MTMD_VIDEO OFF CACHE BOOL "Frames decoded by bounded media adapter" FORCE)
 set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE)
 add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
 add_executable(searchi-gemma-worker worker.cpp)
+if(MINGW)
+    # Windows argument conversion uses wmain; keep the same wide CRT entry.
+    target_link_options(searchi-gemma-worker PRIVATE -municode)
+endif()
 target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
 target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd ${CMAKE_DL_LIBS})
 if(SEARCHI_CPU_DISPATCH)

crates/searchi-semantic/native/worker.cpp

@@ -15,16 +15,17 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
-#ifdef SEARCHI_CPU_DISPATCH
-#include <filesystem>
 #ifdef _WIN32
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
 #include <windows.h>
-#elif defined(__linux__)
+#endif
+#ifdef SEARCHI_CPU_DISPATCH
+#include <filesystem>
+#if defined(__linux__)
 #include <dlfcn.h>
 #include <unistd.h>
-#else
+#elif !defined(_WIN32)
 #error Unsupported platform for packaged CPU dispatch
 #endif
 #endif
@@ -384,7 +385,7 @@ public:
     }
 };
 
-int main(int argc, char ** argv) {
+static int worker_main(int argc, const char * const * argv) {
     try {
         std::string path, device = "auto", role = "indexing"; int threads = 1;
         for (int i = 1; i < argc; ++i) {
@@ -447,3 +448,35 @@ int main(int argc, char ** argv) {
         return 1;
     }
 }
+
+#ifdef _WIN32
+// The narrow CRT argv uses the system ANSI code page (commonly1252), whereas
+// ggml_fopen requires UTF-8. Preserve Windows UTF-16 arguments before parsing;
+// model/runtime directories may contain any valid Unicode user/folder name.
+int wmain(int argc, wchar_t ** argv) {
+    try {
+        std::vector<std::string> utf8;
+        utf8.reserve(size_t(argc));
+        for (int i = 0; i < argc; ++i) {
+            int bytes = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, argv[i], -1, nullptr, 0, nullptr, nullptr);
+            require(bytes > 0, "Worker argument contains invalid Windows Unicode");
+            std::string argument(size_t(bytes), '\0');
+            require(WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, argv[i], -1, argument.data(), bytes, nullptr, nullptr) == bytes,
+                "Cannot convert worker argument to UTF-8");
+            argument.pop_back(); // Conversion includes the terminating NUL.
+            utf8.push_back(std::move(argument));
+        }
+        std::vector<const char *> arguments;
+        arguments.reserve(utf8.size());
+        for (const auto & argument : utf8) arguments.push_back(argument.c_str());
+        return worker_main(argc, arguments.data());
+    } catch (const std::exception & error) {
+        std::cerr << "Searchi Gemma worker: " << error.what() << '\n';
+        return 1;
+    }
+}
+#else
+int main(int argc, char ** argv) {
+    return worker_main(argc, argv);
+}
+#endif

crates/searchi-semantic/tests/unicode_windows_cpu.rs

@@ -0,0 +1,75 @@
+//! Real Windows filename/CRT boundary, independent of the active ANSI code page.
+#![cfg(windows)]
+use anyhow::{Context, Result};
+use searchi_semantic::{Device, GemmaEmbedder};
+use std::path::PathBuf;
+
+#[test]
+#[ignore = "Requires verified Google model and current Windows CPU-capable worker"]
+fn unicode_model_directory_preserves_query_and_indexing_roles() -> Result<()> {
+    let models = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set models"))
+        .canonicalize()?;
+    let worker = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker"))
+        .canonicalize()?;
+    // TempDir owns only new names. Create on the source volume so the fixture
+    // can hardlink the verified backbone without copying or changing its bytes.
+    let fixture = tempfile::Builder::new()
+        .prefix("searchi-unicode-")
+        .tempdir_in(models.parent().context("Model directory needs parent")?)?;
+    let unicode = fixture.path().join("Gemäß_日本_😀");
+    std::fs::create_dir(&unicode)?;
+    let filename = "embeddinggemma-2-Q8_0.gguf";
+    std::fs::hard_link(models.join(filename), unicode.join(filename))?;
+    // Relocate only adjacent runtime files as hardlinks. Windows DLL loading
+    // and packaged backend discovery must also survive a Unicode install path.
+    for entry in std::fs::read_dir(worker.parent().context("Worker needs parent")?)? {
+        let entry = entry?;
+        let path = entry.path();
+        if entry.file_type()?.is_file()
+            && (path
+                .extension()
+                .is_some_and(|ext| ext.eq_ignore_ascii_case("dll"))
+                || path == worker)
+        {
+            std::fs::hard_link(&path, unicode.join(entry.file_name()))?;
+        }
+    }
+    let relocated_worker = unicode.join(worker.file_name().context("Worker needs filename")?);
+    let mut reference = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
+    let expected = reference.embed_query("cancel the supplier partnership")?;
+    drop(reference);
+    let mut query =
+        GemmaEmbedder::load_query_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
+    assert_eq!(query.info.role, "query");
+    assert_eq!(query.info.context_tokens, 512);
+    assert_eq!(query.info.device, "cpu");
+    let actual = query.embed_query("cancel the supplier partnership")?;
+    let cosine: f64 = actual
+        .iter()
+        .zip(&expected)
+        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+        .sum();
+    assert!(
+        cosine >= 0.99999,
+        "Unicode path changed inference: {cosine}"
+    );
+    drop(query);
+    let mut indexing =
+        GemmaEmbedder::load_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
+    assert_eq!(indexing.info.role, "indexing");
+    assert_eq!(indexing.info.context_tokens, 2048);
+    assert_eq!(
+        indexing
+            .embed_documents(
+                &["A supplier agreement with sixty days written notice."],
+                "Supplier"
+            )?
+            .len(),
+        1
+    );
+    drop(indexing);
+    drop(fixture); // Hardlink deletion does not remove the original model.
+    assert_eq!(std::fs::metadata(models.join(filename))?.len(), 309855456);
+    println!("Real Unicode model query/indexing roles passed; ASCII/Unicode cosine {cosine:.8}");
+    Ok(())
+}

docs/WINDOWS-UNICODE.md

@@ -0,0 +1,62 @@
+# Windows Unicode model paths
+
+The native worker must preserve filenames from Windows command-line arguments.
+Windows narrow CRT `char** argv` uses the active ANSI code page; the pinned
+llama/ggml model loader expects UTF-8 filenames. UTF-8 JSON requests alone do
+not correct the model path supplied at startup.
+
+## Observed failure, 2026-10-13
+
+An actual CPU query-role startup used a hardlink to the verified Google backbone
+under `Gemäß_日本_😀`. The model bytes were unchanged (309,855,456 bytes,
+SHA256 `2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135`).
+On this Windows VM the active ANSI code page was 1252. The normal worker with
+SHA256 `1272797af94c21db1ccabaa3ad996ab57d64f1bf9d33280c61d65c51360deab7`
+exited with code 1 before a ready handshake. `gguf_init_from_file` reported an
+invalid argument with the Japanese and emoji components replaced by question
+marks, and the German component no longer valid UTF-8. The captured raw stderr
+is preserved locally in `.searchi/qa/windows-unicode-model-before-stderr.bin`.
+
+## Implementation
+
+Windows enters through `wmain`, converts the original UTF-16 arguments to UTF-8
+with `WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS)`, then calls the shared
+worker entry. Invalid Unicode fails explicitly. Unix continues to pass its
+existing byte arguments through `main`. Normal CPU/GPU selection, protocol,
+model validation and role behavior remain unchanged. MinGW receives its required
+`-municode` link option, but a MinGW build has not been executed or certified.
+
+After the fix, normal MSVC worker SHA256
+`c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73`
+passed the real Unicode query/indexing regression (21.03 s). A strengthened
+regression also relocated the worker and every adjacent DLL as hardlinks under
+the Unicode folder: actual query and full document indexing passed (22.45 s),
+with ASCII/Unicode query-vector cosine 1.00000001. The temporary fixture names
+were verified absent afterward and the original backbone remained intact.
+The unchanged query-context/parity/boundary regression passed again (25.83 s).
+
+A separate native after-probe reported query role, 512-token context, CPU,
+`ggml-cpu-cascadelake.dll`, no Vulkan module, and a normalized 768-dimensional
+vector. Metadata is retained in `.searchi/qa/windows-unicode-model-after.json`.
+The ad-hoc model hardlink was removed only after verifying its absolute path
+under the owned QA root and that the directory contained exactly that link.
+The original verified model remains present. No GPU was executed.
+
+The actual regression creates a uniquely owned temporary directory on the
+model's volume, hardlinks the backbone under a Unicode child, relocates adjacent
+runtime files as hardlinks, and compares
+ASCII/Unicode CPU query vectors. It also opens the full indexing role through
+that Unicode path and encodes a real document. Temporary-name cleanup leaves
+the original verified model intact. No duplicate model download or copy is
+needed; hardware acceleration is not executed in this VM.
+
+```powershell
+$env:SEARCHI_FORCE_CPU = '1'
+$env:SEARCHI_GEMMA_MODEL_DIR = 'PATH/TO/VERIFIED/MODELS'
+$env:SEARCHI_GEMMA_WORKER = 'PATH/TO/CURRENT/searchi-gemma-worker.exe'
+cargo test -p searchi-semantic --test unicode_windows_cpu --locked -- --ignored --nocapture
+```
+
+Native package restaging and installed Unicode-profile paths require their own
+checks. This focused model-path regression does not certify every filesystem,
+codec, launcher or installer path, or any GPU hardware.