build: package universal Windows runtime and verify automatic CPU fallback
This commit is contained in:
1 parent
0fcc8ec21c
commit
278b09f82c
16 files changed
+695
-34
No files matched your search
@@ -48,6 +48,8 @@ class Encoder {
|
||||
}
|
||||
bool open(bool gpu) {
|
||||
auto mp = llama_model_default_params();
|
||||
static ggml_backend_dev_t cpu_devices[] = {nullptr};
|
||||
if (!gpu) mp.devices = cpu_devices;
|
||||
mp.n_gpu_layers = gpu ? 999 : 0;
|
||||
mp.progress_callback = [](float,void *) {return true;};
|
||||
model = llama_model_load_from_file(model_path.c_str(),mp);
|
||||
@@ -55,6 +57,7 @@ class Encoder {
|
||||
auto cp = llama_context_default_params();
|
||||
cp.n_ctx = context_tokens; cp.n_batch = context_tokens; cp.n_ubatch = context_tokens;
|
||||
cp.n_seq_max = 1; cp.kv_unified = true; cp.embeddings = true;
|
||||
cp.offload_kqv = gpu; cp.op_offload = gpu;
|
||||
cp.pooling_type = LLAMA_POOLING_TYPE_MEAN;
|
||||
cp.n_threads = threads; cp.n_threads_batch = threads;
|
||||
context = llama_init_from_model(model,cp);
|
||||
@@ -327,6 +330,16 @@ int main(int argc, char ** argv) {
|
||||
require(!path.empty() && threads >= 1 && threads <= 8,"Model path required; threads must be 1–8");
|
||||
require(device == "cpu" || device == "auto","Device must be cpu or auto");
|
||||
bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1";
|
||||
// Disable backend registration before llama initializes its registry.
|
||||
// A GPU-capable distribution can therefore be tested on CPU without
|
||||
// probing/enabling the Vulkan provider in the Windows VM.
|
||||
if (force_cpu || device == "cpu") {
|
||||
#ifdef _WIN32
|
||||
require(_putenv_s("GGML_DISABLE_VULKAN", "1") == 0, "Cannot disable Vulkan for CPU startup");
|
||||
#else
|
||||
require(setenv("GGML_DISABLE_VULKAN", "1", 1) == 0, "Cannot disable Vulkan for CPU startup");
|
||||
#endif
|
||||
}
|
||||
Encoder encoder(path,threads,device == "auto" && !force_cpu);
|
||||
std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
|
||||
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true},{"code_query_supported",true}}).dump() << '\n' << std::flush;
|
||||
|
||||
Reference in new issue
Block a user