Ship portable CPU dispatch and verified shared runtime dependencies
This commit is contained in:
karl committed
1 parent
278b09f82c
commit
0bba1a0aa7
19 files changed
+675
-17
No files matched your search
@@ -4,10 +4,28 @@ if(NOT DEFINED LLAMA_CPP_DIR OR NOT EXISTS "${LLAMA_CPP_DIR}/include/llama.h")
|
||||
message(FATAL_ERROR "Supply -DLLAMA_CPP_DIR=<pinned patched llama.cpp b11461 source>")
|
||||
endif()
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
if(MSVC)
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>" CACHE STRING "Standalone worker static MSVC runtime" FORCE)
|
||||
# CPU-only builds remain small internal VM tools. Normal Windows x64 builds
|
||||
# ship baseline and optimized CPU modules in the same GPU-capable package.
|
||||
set(SEARCHI_ACCELERATION "auto" CACHE STRING "auto; explicit developer overrides: cpu, metal, vulkan, cuda")
|
||||
set(SEARCHI_CPU_DISPATCH OFF)
|
||||
if(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND
|
||||
(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$" OR CMAKE_GENERATOR_PLATFORM STREQUAL "x64") AND
|
||||
NOT SEARCHI_ACCELERATION STREQUAL "cpu")
|
||||
set(SEARCHI_CPU_DISPATCH ON)
|
||||
endif()
|
||||
set(BUILD_SHARED_LIBS OFF CACHE BOOL "Static local worker runtime" FORCE)
|
||||
if(MSVC)
|
||||
# llama and ggml exchange FILE* and other CRT-owned state across DLLs.
|
||||
# Every shared-runtime component must use the same dynamic CRT.
|
||||
if(SEARCHI_CPU_DISPATCH)
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL" CACHE STRING "Shared worker CRT across runtime DLLs" FORCE)
|
||||
else()
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>" CACHE STRING "Internal static worker CRT" FORCE)
|
||||
endif()
|
||||
endif()
|
||||
set(BUILD_SHARED_LIBS ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Shared runtime for Windows CPU dispatch" FORCE)
|
||||
set(GGML_BACKEND_DL ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Load compatible packaged CPU backend" FORCE)
|
||||
set(GGML_CPU_ALL_VARIANTS ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Baseline and optimized CPU variants" FORCE)
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
|
||||
set(GGML_OPENMP OFF CACHE BOOL "Use bounded native llama thread pools without external OpenMP runtime" FORCE)
|
||||
set(GGML_NATIVE OFF CACHE BOOL "Portable CPU kernels")
|
||||
# Normal distribution builds include GPU support and CPU fallback. CPU-only
|
||||
@@ -15,7 +33,6 @@ set(GGML_NATIVE OFF CACHE BOOL "Portable CPU kernels")
|
||||
set(GGML_CUDA OFF CACHE BOOL "Production CUDA backend")
|
||||
set(GGML_VULKAN OFF CACHE BOOL "Production Vulkan backend")
|
||||
set(GGML_METAL OFF CACHE BOOL "Production Metal backend")
|
||||
set(SEARCHI_ACCELERATION "auto" CACHE STRING "auto; explicit developer overrides: cpu, metal, vulkan, cuda")
|
||||
if(SEARCHI_ACCELERATION STREQUAL "cpu")
|
||||
set(GGML_CUDA OFF CACHE BOOL "CPU test build" FORCE)
|
||||
set(GGML_VULKAN OFF CACHE BOOL "CPU test build" FORCE)
|
||||
@@ -46,5 +63,8 @@ add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
|
||||
add_executable(searchi-gemma-worker worker.cpp)
|
||||
target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
|
||||
target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd)
|
||||
if(SEARCHI_CPU_DISPATCH)
|
||||
target_compile_definitions(searchi-gemma-worker PRIVATE SEARCHI_CPU_DISPATCH=1)
|
||||
endif()
|
||||
target_compile_definitions(searchi-gemma-worker PRIVATE SEARCHI_LLAMA_TAG="b11461-no-logits-v1")
|
||||
install(TARGETS searchi-gemma-worker RUNTIME DESTINATION bin)
|
||||
@@ -1,6 +1,7 @@
|
||||
// Local NDJSON helper around the pinned llama.cpp encoder. stdout is exclusively
|
||||
// protocol output; llama diagnostics go to stderr. No model download or HTTP.
|
||||
#include "llama.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "mtmd.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
#include <algorithm>
|
||||
@@ -14,6 +15,12 @@
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#ifdef SEARCHI_CPU_DISPATCH
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#define NOMINMAX
|
||||
#include <windows.h>
|
||||
#include <filesystem>
|
||||
#endif
|
||||
|
||||
using json = nlohmann::json;
|
||||
constexpr size_t max_request_bytes = 16 * 1024 * 1024;
|
||||
@@ -71,6 +78,8 @@ class Encoder {
|
||||
}
|
||||
public:
|
||||
std::string device;
|
||||
std::string cpu_backend_module;
|
||||
bool vulkan_module_loaded = false;
|
||||
int threads;
|
||||
Encoder(const std::string & path, int cpu_threads, bool want_gpu) : model_path(path),threads(cpu_threads) {
|
||||
llama_log_set([](ggml_log_level level, const char * text, void *) {
|
||||
@@ -79,6 +88,31 @@ public:
|
||||
mtmd_log_set([](ggml_log_level level, const char * text, void *) {
|
||||
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text,stderr);
|
||||
},nullptr);
|
||||
#ifdef SEARCHI_CPU_DISPATCH
|
||||
// Only load our adjacent modules: no current-directory or environment
|
||||
// plugin discovery. CPU-only startup never loads the Vulkan module.
|
||||
std::vector<wchar_t> executable(32768);
|
||||
DWORD length = GetModuleFileNameW(nullptr, executable.data(), DWORD(executable.size()));
|
||||
require(length > 0 && length < executable.size(), "Cannot locate packaged worker backends");
|
||||
const auto directory = std::filesystem::path(std::wstring(executable.data(), length)).parent_path().u8string();
|
||||
if (want_gpu) {
|
||||
ggml_backend_load_all_from_path(directory.c_str());
|
||||
} else {
|
||||
ggml_backend_load_cpu_only_from_path(directory.c_str());
|
||||
}
|
||||
require(ggml_backend_reg_by_name("CPU") != nullptr, "Cannot load compatible packaged CPU backend");
|
||||
const auto cpu_reg = ggml_backend_reg_by_name("CPU");
|
||||
const auto feature_proc = ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_get_features");
|
||||
HMODULE cpu_module = nullptr;
|
||||
if (feature_proc && GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
|
||||
reinterpret_cast<LPCWSTR>(feature_proc), &cpu_module)) {
|
||||
DWORD module_length = GetModuleFileNameW(cpu_module, executable.data(), DWORD(executable.size()));
|
||||
if (module_length > 0 && module_length < executable.size()) {
|
||||
cpu_backend_module = std::filesystem::path(std::wstring(executable.data(), module_length)).filename().u8string();
|
||||
}
|
||||
}
|
||||
vulkan_module_loaded = GetModuleHandleW(L"ggml-vulkan.dll") != nullptr;
|
||||
#endif
|
||||
llama_backend_init();
|
||||
bool use_gpu = want_gpu && llama_supports_gpu_offload();
|
||||
require(open(use_gpu) || (use_gpu && open(false)),"Cannot initialize local EmbeddingGemma 2 model/context");
|
||||
@@ -335,14 +369,17 @@ int main(int argc, char ** argv) {
|
||||
// probing/enabling the Vulkan provider in the Windows VM.
|
||||
if (force_cpu || device == "cpu") {
|
||||
#ifdef _WIN32
|
||||
require(_putenv_s("GGML_CPU_ONLY", "1") == 0, "Cannot restrict CPU startup");
|
||||
require(_putenv_s("GGML_DISABLE_VULKAN", "1") == 0, "Cannot disable Vulkan for CPU startup");
|
||||
#else
|
||||
require(setenv("GGML_CPU_ONLY", "1", 1) == 0, "Cannot restrict CPU startup");
|
||||
require(setenv("GGML_DISABLE_VULKAN", "1", 1) == 0, "Cannot disable Vulkan for CPU startup");
|
||||
#endif
|
||||
}
|
||||
Encoder encoder(path,threads,device == "auto" && !force_cpu);
|
||||
std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
|
||||
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true},{"code_query_supported",true}}).dump() << '\n' << std::flush;
|
||||
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true},{"code_query_supported",true},
|
||||
{"cpu_backend_module",encoder.cpu_backend_module},{"vulkan_module_loaded",encoder.vulkan_module_loaded}}).dump() << '\n' << std::flush;
|
||||
for (;;) {
|
||||
std::string line; char c;
|
||||
while (std::cin.get(c) && c != '\n') {
|
||||
|
||||
Reference in new issue
Block a user