Preserve Unicode Windows arguments when loading native models

crates/searchi-semantic/Cargo.toml

@@ -21,4 +21,5 @@ serde.workspace = true
 serde_json.workspace = true
 
 [dev-dependencies]
+tempfile = "3"
 serde_json.workspace = true

crates/searchi-semantic/native/CMakeLists.txt

@@ -67,6 +67,10 @@ set(MTMD_VIDEO OFF CACHE BOOL "Frames decoded by bounded media adapter" FORCE)
 set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE)
 add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
 add_executable(searchi-gemma-worker worker.cpp)
+if(MINGW)
+    # Windows argument conversion uses wmain; keep the same wide CRT entry.
+    target_link_options(searchi-gemma-worker PRIVATE -municode)
+endif()
 target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
 target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd ${CMAKE_DL_LIBS})
 if(SEARCHI_CPU_DISPATCH)

crates/searchi-semantic/native/worker.cpp

@@ -15,16 +15,17 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
-#ifdef SEARCHI_CPU_DISPATCH
-#include <filesystem>
 #ifdef _WIN32
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
 #include <windows.h>
-#elif defined(__linux__)
+#endif
+#ifdef SEARCHI_CPU_DISPATCH
+#include <filesystem>
+#if defined(__linux__)
 #include <dlfcn.h>
 #include <unistd.h>
-#else
+#elif !defined(_WIN32)
 #error Unsupported platform for packaged CPU dispatch
 #endif
 #endif
@@ -384,7 +385,7 @@ public:
     }
 };
 
-int main(int argc, char ** argv) {
+static int worker_main(int argc, const char * const * argv) {
     try {
         std::string path, device = "auto", role = "indexing"; int threads = 1;
         for (int i = 1; i < argc; ++i) {
@@ -447,3 +448,35 @@ int main(int argc, char ** argv) {
         return 1;
     }
 }
+
+#ifdef _WIN32
+// The narrow CRT argv uses the system ANSI code page (commonly1252), whereas
+// ggml_fopen requires UTF-8. Preserve Windows UTF-16 arguments before parsing;
+// model/runtime directories may contain any valid Unicode user/folder name.
+int wmain(int argc, wchar_t ** argv) {
+    try {
+        std::vector<std::string> utf8;
+        utf8.reserve(size_t(argc));
+        for (int i = 0; i < argc; ++i) {
+            int bytes = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, argv[i], -1, nullptr, 0, nullptr, nullptr);
+            require(bytes > 0, "Worker argument contains invalid Windows Unicode");
+            std::string argument(size_t(bytes), '\0');
+            require(WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, argv[i], -1, argument.data(), bytes, nullptr, nullptr) == bytes,
+                "Cannot convert worker argument to UTF-8");
+            argument.pop_back(); // Conversion includes the terminating NUL.
+            utf8.push_back(std::move(argument));
+        }
+        std::vector<const char *> arguments;
+        arguments.reserve(utf8.size());
+        for (const auto & argument : utf8) arguments.push_back(argument.c_str());
+        return worker_main(argc, arguments.data());
+    } catch (const std::exception & error) {
+        std::cerr << "Searchi Gemma worker: " << error.what() << '\n';
+        return 1;
+    }
+}
+#else
+int main(int argc, char ** argv) {
+    return worker_main(argc, argv);
+}
+#endif

crates/searchi-semantic/tests/unicode_windows_cpu.rs

@@ -0,0 +1,75 @@
+//! Real Windows filename/CRT boundary, independent of the active ANSI code page.
+#![cfg(windows)]
+use anyhow::{Context, Result};
+use searchi_semantic::{Device, GemmaEmbedder};
+use std::path::PathBuf;
+
+#[test]
+#[ignore = "Requires verified Google model and current Windows CPU-capable worker"]
+fn unicode_model_directory_preserves_query_and_indexing_roles() -> Result<()> {
+    let models = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set models"))
+        .canonicalize()?;
+    let worker = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker"))
+        .canonicalize()?;
+    // TempDir owns only new names. Create on the source volume so the fixture
+    // can hardlink the verified backbone without copying or changing its bytes.
+    let fixture = tempfile::Builder::new()
+        .prefix("searchi-unicode-")
+        .tempdir_in(models.parent().context("Model directory needs parent")?)?;
+    let unicode = fixture.path().join("Gemäß_日本_😀");
+    std::fs::create_dir(&unicode)?;
+    let filename = "embeddinggemma-2-Q8_0.gguf";
+    std::fs::hard_link(models.join(filename), unicode.join(filename))?;
+    // Relocate only adjacent runtime files as hardlinks. Windows DLL loading
+    // and packaged backend discovery must also survive a Unicode install path.
+    for entry in std::fs::read_dir(worker.parent().context("Worker needs parent")?)? {
+        let entry = entry?;
+        let path = entry.path();
+        if entry.file_type()?.is_file()
+            && (path
+                .extension()
+                .is_some_and(|ext| ext.eq_ignore_ascii_case("dll"))
+                || path == worker)
+        {
+            std::fs::hard_link(&path, unicode.join(entry.file_name()))?;
+        }
+    }
+    let relocated_worker = unicode.join(worker.file_name().context("Worker needs filename")?);
+    let mut reference = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
+    let expected = reference.embed_query("cancel the supplier partnership")?;
+    drop(reference);
+    let mut query =
+        GemmaEmbedder::load_query_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
+    assert_eq!(query.info.role, "query");
+    assert_eq!(query.info.context_tokens, 512);
+    assert_eq!(query.info.device, "cpu");
+    let actual = query.embed_query("cancel the supplier partnership")?;
+    let cosine: f64 = actual
+        .iter()
+        .zip(&expected)
+        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+        .sum();
+    assert!(
+        cosine >= 0.99999,
+        "Unicode path changed inference: {cosine}"
+    );
+    drop(query);
+    let mut indexing =
+        GemmaEmbedder::load_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
+    assert_eq!(indexing.info.role, "indexing");
+    assert_eq!(indexing.info.context_tokens, 2048);
+    assert_eq!(
+        indexing
+            .embed_documents(
+                &["A supplier agreement with sixty days written notice."],
+                "Supplier"
+            )?
+            .len(),
+        1
+    );
+    drop(indexing);
+    drop(fixture); // Hardlink deletion does not remove the original model.
+    assert_eq!(std::fs::metadata(models.join(filename))?.len(), 309855456);
+    println!("Real Unicode model query/indexing roles passed; ASCII/Unicode cosine {cosine:.8}");
+    Ok(())
+}