Preserve Unicode Windows arguments when loading native models

crates/searchi-semantic/tests/unicode_windows_cpu.rs

@@ -0,0 +1,75 @@
+//! Real Windows filename/CRT boundary, independent of the active ANSI code page.
+#![cfg(windows)]
+use anyhow::{Context, Result};
+use searchi_semantic::{Device, GemmaEmbedder};
+use std::path::PathBuf;
+
+#[test]
+#[ignore = "Requires verified Google model and current Windows CPU-capable worker"]
+fn unicode_model_directory_preserves_query_and_indexing_roles() -> Result<()> {
+    let models = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set models"))
+        .canonicalize()?;
+    let worker = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker"))
+        .canonicalize()?;
+    // TempDir owns only new names. Create on the source volume so the fixture
+    // can hardlink the verified backbone without copying or changing its bytes.
+    let fixture = tempfile::Builder::new()
+        .prefix("searchi-unicode-")
+        .tempdir_in(models.parent().context("Model directory needs parent")?)?;
+    let unicode = fixture.path().join("Gemäß_日本_😀");
+    std::fs::create_dir(&unicode)?;
+    let filename = "embeddinggemma-2-Q8_0.gguf";
+    std::fs::hard_link(models.join(filename), unicode.join(filename))?;
+    // Relocate only adjacent runtime files as hardlinks. Windows DLL loading
+    // and packaged backend discovery must also survive a Unicode install path.
+    for entry in std::fs::read_dir(worker.parent().context("Worker needs parent")?)? {
+        let entry = entry?;
+        let path = entry.path();
+        if entry.file_type()?.is_file()
+            && (path
+                .extension()
+                .is_some_and(|ext| ext.eq_ignore_ascii_case("dll"))
+                || path == worker)
+        {
+            std::fs::hard_link(&path, unicode.join(entry.file_name()))?;
+        }
+    }
+    let relocated_worker = unicode.join(worker.file_name().context("Worker needs filename")?);
+    let mut reference = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
+    let expected = reference.embed_query("cancel the supplier partnership")?;
+    drop(reference);
+    let mut query =
+        GemmaEmbedder::load_query_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
+    assert_eq!(query.info.role, "query");
+    assert_eq!(query.info.context_tokens, 512);
+    assert_eq!(query.info.device, "cpu");
+    let actual = query.embed_query("cancel the supplier partnership")?;
+    let cosine: f64 = actual
+        .iter()
+        .zip(&expected)
+        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+        .sum();
+    assert!(
+        cosine >= 0.99999,
+        "Unicode path changed inference: {cosine}"
+    );
+    drop(query);
+    let mut indexing =
+        GemmaEmbedder::load_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
+    assert_eq!(indexing.info.role, "indexing");
+    assert_eq!(indexing.info.context_tokens, 2048);
+    assert_eq!(
+        indexing
+            .embed_documents(
+                &["A supplier agreement with sixty days written notice."],
+                "Supplier"
+            )?
+            .len(),
+        1
+    );
+    drop(indexing);
+    drop(fixture); // Hardlink deletion does not remove the original model.
+    assert_eq!(std::fs::metadata(models.join(filename))?.len(), 309855456);
+    println!("Real Unicode model query/indexing roles passed; ASCII/Unicode cosine {cosine:.8}");
+    Ok(())
+}