75 lines
3.2 KiB
Rust
75 lines
3.2 KiB
Rust
|
//! Real Windows filename/CRT boundary, independent of the active ANSI code page.
|
||
|
|
#![cfg(windows)]
|
||
|
|
use anyhow::{Context, Result};
|
||
|
|
use searchi_semantic::{Device, GemmaEmbedder};
|
||
|
|
use std::path::PathBuf;
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
#[ignore = "Requires verified Google model and current Windows CPU-capable worker"]
|
||
|
|
fn unicode_model_directory_preserves_query_and_indexing_roles() -> Result<()> {
|
||
|
|
let models = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set models"))
|
||
|
|
.canonicalize()?;
|
||
|
|
let worker = PathBuf::from(std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker"))
|
||
|
|
.canonicalize()?;
|
||
|
|
// TempDir owns only new names. Create on the source volume so the fixture
|
||
|
|
// can hardlink the verified backbone without copying or changing its bytes.
|
||
|
|
let fixture = tempfile::Builder::new()
|
||
|
|
.prefix("searchi-unicode-")
|
||
|
|
.tempdir_in(models.parent().context("Model directory needs parent")?)?;
|
||
|
|
let unicode = fixture.path().join("Gemäß_日本_😀");
|
||
|
|
std::fs::create_dir(&unicode)?;
|
||
|
|
let filename = "embeddinggemma-2-Q8_0.gguf";
|
||
|
|
std::fs::hard_link(models.join(filename), unicode.join(filename))?;
|
||
|
|
// Relocate only adjacent runtime files as hardlinks. Windows DLL loading
|
||
|
|
// and packaged backend discovery must also survive a Unicode install path.
|
||
|
|
for entry in std::fs::read_dir(worker.parent().context("Worker needs parent")?)? {
|
||
|
|
let entry = entry?;
|
||
|
|
let path = entry.path();
|
||
|
|
if entry.file_type()?.is_file()
|
||
|
|
&& (path
|
||
|
|
.extension()
|
||
|
|
.is_some_and(|ext| ext.eq_ignore_ascii_case("dll"))
|
||
|
|
|| path == worker)
|
||
|
|
{
|
||
|
|
std::fs::hard_link(&path, unicode.join(entry.file_name()))?;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let relocated_worker = unicode.join(worker.file_name().context("Worker needs filename")?);
|
||
|
|
let mut reference = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
|
||
|
|
let expected = reference.embed_query("cancel the supplier partnership")?;
|
||
|
|
drop(reference);
|
||
|
|
let mut query =
|
||
|
|
GemmaEmbedder::load_query_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
|
||
|
|
assert_eq!(query.info.role, "query");
|
||
|
|
assert_eq!(query.info.context_tokens, 512);
|
||
|
|
assert_eq!(query.info.device, "cpu");
|
||
|
|
let actual = query.embed_query("cancel the supplier partnership")?;
|
||
|
|
let cosine: f64 = actual
|
||
|
|
.iter()
|
||
|
|
.zip(&expected)
|
||
|
|
.map(|(a, b)| f64::from(*a) * f64::from(*b))
|
||
|
|
.sum();
|
||
|
|
assert!(
|
||
|
|
cosine >= 0.99999,
|
||
|
|
"Unicode path changed inference: {cosine}"
|
||
|
|
);
|
||
|
|
drop(query);
|
||
|
|
let mut indexing =
|
||
|
|
GemmaEmbedder::load_with_device(&unicode, &relocated_worker, 1, Device::Cpu)?;
|
||
|
|
assert_eq!(indexing.info.role, "indexing");
|
||
|
|
assert_eq!(indexing.info.context_tokens, 2048);
|
||
|
|
assert_eq!(
|
||
|
|
indexing
|
||
|
|
.embed_documents(
|
||
|
|
&["A supplier agreement with sixty days written notice."],
|
||
|
|
"Supplier"
|
||
|
|
)?
|
||
|
|
.len(),
|
||
|
|
1
|
||
|
|
);
|
||
|
|
drop(indexing);
|
||
|
|
drop(fixture); // Hardlink deletion does not remove the original model.
|
||
|
|
assert_eq!(std::fs::metadata(models.join(filename))?.len(), 309855456);
|
||
|
|
println!("Real Unicode model query/indexing roles passed; ASCII/Unicode cosine {cosine:.8}");
|
||
|
|
Ok(())
|
||
|
|
}
|