Package portable Linux runtime and verify Google CPU search
This commit is contained in:
1 parent
e3558a0104
commit
0b56ab553c
22 files changed
+1585
-16
No files matched your search
@@ -4,11 +4,11 @@ if(NOT DEFINED LLAMA_CPP_DIR OR NOT EXISTS "${LLAMA_CPP_DIR}/include/llama.h")
|
||||
message(FATAL_ERROR "Supply -DLLAMA_CPP_DIR=<pinned patched llama.cpp b11461 source>")
|
||||
endif()
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
# CPU-only builds remain small internal VM tools. Normal Windows x64 builds
|
||||
# CPU-only builds remain small internal VM tools. Normal Windows/Linux x64 builds
|
||||
# ship baseline and optimized CPU modules in the same GPU-capable package.
|
||||
set(SEARCHI_ACCELERATION "auto" CACHE STRING "auto; explicit developer overrides: cpu, metal, vulkan, cuda")
|
||||
set(SEARCHI_CPU_DISPATCH OFF)
|
||||
if(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND
|
||||
if((WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND SEARCHI_ACCELERATION MATCHES "^(auto|vulkan)$")) AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND
|
||||
(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$" OR CMAKE_GENERATOR_PLATFORM STREQUAL "x64") AND
|
||||
NOT SEARCHI_ACCELERATION STREQUAL "cpu")
|
||||
set(SEARCHI_CPU_DISPATCH ON)
|
||||
@@ -22,12 +22,18 @@ if(MSVC)
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>" CACHE STRING "Internal static worker CRT" FORCE)
|
||||
endif()
|
||||
endif()
|
||||
set(BUILD_SHARED_LIBS ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Shared runtime for Windows CPU dispatch" FORCE)
|
||||
set(BUILD_SHARED_LIBS ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Shared runtime for portable CPU dispatch" FORCE)
|
||||
set(GGML_BACKEND_DL ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Load compatible packaged CPU backend" FORCE)
|
||||
set(GGML_CPU_ALL_VARIANTS ${SEARCHI_CPU_DISPATCH} CACHE BOOL "Baseline and optimized CPU variants" FORCE)
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND SEARCHI_CPU_DISPATCH)
|
||||
# Adjacent packaged libraries, independent of the build tree or cwd.
|
||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
|
||||
set(CMAKE_INSTALL_RPATH "$ORIGIN")
|
||||
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH OFF)
|
||||
endif()
|
||||
set(GGML_OPENMP OFF CACHE BOOL "Use bounded native llama thread pools without external OpenMP runtime" FORCE)
|
||||
set(GGML_NATIVE OFF CACHE BOOL "Portable CPU kernels")
|
||||
set(GGML_NATIVE OFF CACHE BOOL "Portable CPU kernels" FORCE)
|
||||
# Normal distribution builds include GPU support and CPU fallback. CPU-only
|
||||
# configuration is explicitly reserved for development/testing in the VM.
|
||||
set(GGML_CUDA OFF CACHE BOOL "Production CUDA backend")
|
||||
@@ -62,7 +68,7 @@ set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE)
|
||||
add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
|
||||
add_executable(searchi-gemma-worker worker.cpp)
|
||||
target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
|
||||
target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd)
|
||||
target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd ${CMAKE_DL_LIBS})
|
||||
if(SEARCHI_CPU_DISPATCH)
|
||||
target_compile_definitions(searchi-gemma-worker PRIVATE SEARCHI_CPU_DISPATCH=1)
|
||||
endif()
|
||||
|
||||
@@ -16,10 +16,17 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#ifdef SEARCHI_CPU_DISPATCH
|
||||
#include <filesystem>
|
||||
#ifdef _WIN32
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#define NOMINMAX
|
||||
#include <windows.h>
|
||||
#include <filesystem>
|
||||
#elif defined(__linux__)
|
||||
#include <dlfcn.h>
|
||||
#include <unistd.h>
|
||||
#else
|
||||
#error Unsupported platform for packaged CPU dispatch
|
||||
#endif
|
||||
#endif
|
||||
|
||||
using json = nlohmann::json;
|
||||
@@ -91,10 +98,19 @@ public:
|
||||
#ifdef SEARCHI_CPU_DISPATCH
|
||||
// Only load our adjacent modules: no current-directory or environment
|
||||
// plugin discovery. CPU-only startup never loads the Vulkan module.
|
||||
#ifdef _WIN32
|
||||
std::vector<wchar_t> executable(32768);
|
||||
DWORD length = GetModuleFileNameW(nullptr, executable.data(), DWORD(executable.size()));
|
||||
require(length > 0 && length < executable.size(), "Cannot locate packaged worker backends");
|
||||
const auto directory = std::filesystem::path(std::wstring(executable.data(), length)).parent_path().u8string();
|
||||
#else
|
||||
// /proc identifies this executable even when launched through PATH or
|
||||
// from another working directory. Reject truncation before discovery.
|
||||
std::vector<char> executable(32768);
|
||||
const auto length = readlink("/proc/self/exe", executable.data(), executable.size());
|
||||
require(length > 0 && size_t(length) < executable.size(), "Cannot locate packaged worker backends");
|
||||
const auto directory = std::filesystem::path(std::string(executable.data(), size_t(length))).parent_path().string();
|
||||
#endif
|
||||
if (want_gpu) {
|
||||
ggml_backend_load_all_from_path(directory.c_str());
|
||||
} else {
|
||||
@@ -103,6 +119,7 @@ public:
|
||||
require(ggml_backend_reg_by_name("CPU") != nullptr, "Cannot load compatible packaged CPU backend");
|
||||
const auto cpu_reg = ggml_backend_reg_by_name("CPU");
|
||||
const auto feature_proc = ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_get_features");
|
||||
#ifdef _WIN32
|
||||
HMODULE cpu_module = nullptr;
|
||||
if (feature_proc && GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
|
||||
reinterpret_cast<LPCWSTR>(feature_proc), &cpu_module)) {
|
||||
@@ -112,6 +129,19 @@ public:
|
||||
}
|
||||
}
|
||||
vulkan_module_loaded = GetModuleHandleW(L"ggml-vulkan.dll") != nullptr;
|
||||
#else
|
||||
Dl_info cpu_module{};
|
||||
if (feature_proc && dladdr(feature_proc, &cpu_module) && cpu_module.dli_fname) {
|
||||
cpu_backend_module = std::filesystem::path(cpu_module.dli_fname).filename().string();
|
||||
}
|
||||
// RTLD_NOLOAD observes module presence without initiating provider
|
||||
// loading; forced CPU startup must remain independent of Vulkan.
|
||||
const auto vulkan_path = std::filesystem::path(directory) / "libggml-vulkan.so";
|
||||
if (void * module = dlopen(vulkan_path.c_str(), RTLD_NOW | RTLD_NOLOAD)) {
|
||||
vulkan_module_loaded = true;
|
||||
dlclose(module);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
llama_backend_init();
|
||||
bool use_gpu = want_gpu && llama_supports_gpu_offload();
|
||||
|
||||
@@ -97,6 +97,11 @@ These are the current script and CLI interfaces, not evidence of an executed
|
||||
Linux release. On a native Linux build host with Vulkan development libraries
|
||||
and `glslc`, run from the repository root:
|
||||
|
||||
On Ubuntu 24.04, the native worker build prerequisites are `build-essential`,
|
||||
`cmake`, `git`, `pkg-config`, `libvulkan-dev`, `glslc`, and `spirv-headers`.
|
||||
The last package supplies the SPIR-V CMake configuration required by the pinned
|
||||
Vulkan backend. These build tools do not enable GPU execution in tests.
|
||||
|
||||
```sh
|
||||
python3 tools/embeddinggemma2/build_runtime.py --backend auto --jobs 2
|
||||
python3 tools/embeddinggemma2/download.py
|
||||
@@ -114,6 +119,46 @@ This normal worker detects usable acceleration and falls back to CPU. Internal
|
||||
CPU-only model tests may use `--backend cpu` and `SEARCHI_FORCE_CPU=1`; those
|
||||
test artifacts are not a separate distribution edition.
|
||||
|
||||
### Linux headless archive validation
|
||||
|
||||
The Linux x86-64 normal worker layout includes baseline and optimized CPU
|
||||
modules, an optional Vulkan module and adjacent core libraries. The builder
|
||||
audits ELF dependencies and origin-relative lookup before writing a hashed
|
||||
runtime inventory. ARM and other distribution/ABI combinations remain separate
|
||||
release gates.
|
||||
|
||||
After building the normal worker, release CLI and pinned media helpers, prepare
|
||||
an unsigned headless development candidate:
|
||||
|
||||
```sh
|
||||
python3 tools/package_linux.py
|
||||
python3 tools/embeddinggemma2/linux_smoke.py --worker .searchi/runtime/auto/searchi-gemma-worker --models .searchi/models/embeddinggemma2 --output .searchi/packages/linux-worker-smoke.json
|
||||
python3 tools/embeddinggemma2/linux_smoke.py --worker .searchi/runtime/auto/searchi-gemma-worker --models .searchi/models/embeddinggemma2 --output .searchi/packages/linux-baseline-smoke.json --baseline
|
||||
```
|
||||
|
||||
The packager accepts explicit `--executable`, `--runtime`, `--media` and
|
||||
`--llama-license` paths when builds use separate target/cache directories.
|
||||
It packages only approved runtime files, pinned decoders and notices, with
|
||||
normalized archive metadata, executable modes and per-file checksums. It
|
||||
excludes model caches and user indexes/settings. The smoke checks copy the
|
||||
runtime outside its build tree and run genuine Google code inference on CPU;
|
||||
the baseline check omits optimized CPU modules from its temporary extraction.
|
||||
They do not exercise GPU providers or certify older physical CPUs, installers
|
||||
or desktop behavior. Archive extraction and installed CLI search must also
|
||||
be verified before a release gate is closed.
|
||||
|
||||
The optional Linux missing-loader check requires explicit root execution and
|
||||
util-linux mount namespaces/chroot. It never escalates privileges and verifies
|
||||
a private namespace before creating an isolated filesystem:
|
||||
|
||||
```sh
|
||||
sudo python3 tools/embeddinggemma2/linux_loader_smoke.py --worker .searchi/runtime/auto/searchi-gemma-worker --models .searchi/models/embeddinggemma2 --output .searchi/packages/linux-missing-loader.json
|
||||
```
|
||||
|
||||
It runs automatic device selection with no forced-CPU override, but contains no
|
||||
Vulkan loader, driver configuration or GPU devices, so only CPU inference can
|
||||
execute. This is distinct from testing an actual GPU or recovering device loss.
|
||||
|
||||
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
|
||||
command. It provisions model assets, not the native worker. Package the worker
|
||||
separately and pass its path to headless embedding/search. Each short-lived CLI
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# Incremental semantic indexing and recovery
|
||||
|
||||
Status: implementation design and acceptance gates; not implemented. Current
|
||||
semantic rebuilds regenerate vectors and the ANN graph. Interrupted staging is
|
||||
discarded by the next rebuild. The previous ready generation remains searchable.
|
||||
|
||||
## Reuse requires complete sources
|
||||
|
||||
Document vectors depend on the extracted text, the bounded filename title,
|
||||
model, prompt and token-window policy. The current passage content hash alone
|
||||
is insufficient: a basename rename changes Google's document prompt. Passage
|
||||
IDs are recreated by core reconciliation, so reuse must remap current IDs,
|
||||
paths and page/line locations rather than retain old citations.
|
||||
|
||||
Media reuse requires a verified hash of the actual file bytes and matching
|
||||
encoder/preprocessing/sampling policy. Size and modification time alone are
|
||||
insufficient. Existing media chunks are written progressively; their presence
|
||||
does not prove that a file completed. A crash can leave an incomplete group.
|
||||
|
||||
Add a versioned completion manifest per source group: generation, task key,
|
||||
source type, encoder/policy key, source hash, title hash where applicable,
|
||||
expected chunk count and completion state. Associate chunks with their group.
|
||||
Only completed groups can supply reusable vectors. Historical generations
|
||||
without completion evidence remain searchable but require one rebuild before
|
||||
their vectors qualify for reuse.
|
||||
|
||||
Start with matching complete-generation policies. Component encoder keys can
|
||||
later let unchanged document vectors survive an audio-sampling policy change.
|
||||
Do not silently omit image, audio or video work to make a rebuild cheaper.
|
||||
|
||||
## Durable work and atomic publication
|
||||
|
||||
Persist bounded pending/running/done/failed tasks in SQLite. Tasks retain their
|
||||
source identity, attempts and a bounded error. Obtain the existing OS build
|
||||
lock before starting or resuming a compatible staging generation; only then
|
||||
can interrupted running tasks be reset. Keep inference outside write
|
||||
transactions. Commit source completion, verified chunk membership and task
|
||||
state atomically. Incomplete groups must be discarded or resumed without
|
||||
qualifying as completed reuse sources.
|
||||
|
||||
Discover work in bounded keyset batches. Before publication, reconcile staging
|
||||
against current indexed sources: remove deleted sources, invalidate changed
|
||||
identities and enqueue additions. Verify exact document window slices and
|
||||
remap citations. For media, confirm the file is unchanged after extraction and
|
||||
inference. Retain the 100,000-chunk budget and explicit per-file media failures.
|
||||
|
||||
Build the ANN graph only from eligible completed groups. Publish under a
|
||||
transaction that checks the reconciled corpus revision; a concurrent source
|
||||
change must trigger reconciliation/retry. Storage or activation failure must
|
||||
preserve both the previous active generation and the same reader's policy.
|
||||
|
||||
## Acceptance gates
|
||||
|
||||
- [ ] Migrate existing databases while preserving active-generation search.
|
||||
- [ ] Rebuild mixed text/code/image/audio/video fixtures unchanged with zero
|
||||
document/media inference calls and identical vectors and source locations.
|
||||
- [ ] Change one document and one media file; re-encode only affected groups.
|
||||
- [ ] Re-encode text after a basename rename; reuse a folder-only move with
|
||||
current citations. Verify media bytes before reusing a renamed media file.
|
||||
- [ ] Regenerate affected media when sampling policy changes; retain eligible
|
||||
text groups when component encoder-key reuse is implemented.
|
||||
- [ ] Terminate after a committed batch and resume without repeating completed
|
||||
inference. Never reuse or publish an unfinished source group.
|
||||
- [ ] Mutate or delete a queued/running source and prove stale vectors cannot
|
||||
activate or become readable.
|
||||
- [ ] Inject storage/activation failure, preserve existing results and resume.
|
||||
- [ ] Use genuine Google CPU inference for these integration checks. Storage
|
||||
fixtures alone do not establish retrieval quality or reuse equivalence.
|
||||
- [ ] Measure preparation, the first foreground query and warm p95 while
|
||||
resumed indexing runs; record actual inference/reuse counts and resource use.
|
||||
|
||||
Foreground query priority and bounded persistent workers remain part of the
|
||||
first-use requirement. Faster unchanged rebuilds alone do not prove responsive
|
||||
search while indexing.
|
||||
+16
-2
@@ -30,6 +30,10 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
- [ ] Versioned embeddings, ANN index, exact-match priority and hybrid fusion.
|
||||
- [ ] English/German relevance corpus and measured top-10 recall.
|
||||
- [ ] Atomic model/index replacement and resumable persistent work queues.
|
||||
- [ ] Completed-source manifests with title/model/window policy keys and
|
||||
citation remapping; reuse only verified complete document/media groups.
|
||||
- [ ] Durable generation tasks, crash resume and final corpus reconciliation;
|
||||
see [incremental semantic design](INCREMENTAL-SEMANTIC.md).
|
||||
|
||||
## 3. Desktop
|
||||
- [ ] Tauri shell, accessible keyboard search, progressive results and previews (shell built; UI interaction verification pending).
|
||||
@@ -62,6 +66,11 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
- [x] Normal Windows x64 archive includes baseline/optimized CPU dispatch,
|
||||
OS AVX-state guards, shared app-local MSVC CRT and hashed DLL inventory;
|
||||
real optimized and baseline-only package inference passes in the CPU VM.
|
||||
- [x] Normal native Linux x86-64 CPU/Vulkan worker build, hashed 19-library
|
||||
staging and genuine optimized/baseline-only CPU code-query inference outside
|
||||
the build tree; desktop, ABI/older-hardware and GPU validation remain open.
|
||||
- [x] Actual Linux automatic CPU fallback with Vulkan loader, ICDs/drivers and
|
||||
GPU devices absent in a verified private namespace; no GPU provider loaded.
|
||||
- [ ] Verify CPU dispatch on older physical CPUs and a clean Windows machine;
|
||||
dependency inspection and baseline execution on a modern VM are insufficient.
|
||||
- [ ] Actual missing/incompatible GPU drivers, precision, device loss and memory
|
||||
@@ -92,8 +101,13 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
- [ ] Test migrations, interrupted work, invalid documents, inaccessible/offline roots and cloud placeholders.
|
||||
- [ ] Automated native build matrix and documented supported cross-compilation paths.
|
||||
- [x] Native Linux headless workspace tests: 93 passed, 11 explicitly ignored.
|
||||
- [ ] Execute Linux native Google worker and desktop/macOS platform tests;
|
||||
create Windows/macOS/Linux installers.
|
||||
- [x] Native Linux Google text/code/media reference parity and service
|
||||
lifecycle/rebuild recovery: seven genuine CPU integration tests passed.
|
||||
- [x] Unsigned Linux x86-64 headless archive: verify extracted checksums/modes,
|
||||
mixed document/code/audio indexing, decoder discovery, semantic retrieval,
|
||||
line citations and strict filters outside the source/build directories.
|
||||
- [ ] Execute native Linux video/codec coverage and Linux desktop/macOS
|
||||
platform tests; create Windows/macOS/Linux installers.
|
||||
- [ ] Signing/notarization configuration, upgrade/rollback validation, release instructions.
|
||||
|
||||
## 6. Local agents and optional answers
|
||||
|
||||
@@ -44,6 +44,15 @@ external OpenMP runtime. Production CMake supports cpu, auto, metal, vulkan,
|
||||
and cuda; CPU builds force GPU flags off. Runtime/provider parity and packaging
|
||||
on other operating systems remain independent release checks.
|
||||
|
||||
Linux x86-64 normal builds also use shared CPU dispatch: 14 CPU modules,
|
||||
four core libraries and a dynamically loaded Vulkan module. Core SONAME files
|
||||
are copied beside the worker with origin-relative lookup; the builder verifies
|
||||
ELF dependencies and the stager verifies the explicit hashed inventory.
|
||||
Vulkan's system loader is a dependency of the optional GPU module rather than
|
||||
the worker or CPU/core libraries. Native optimized and baseline-only CPU
|
||||
code-query smoke checks pass. System ABI coverage, reference parity, GPU
|
||||
operation and desktop packaging are separate release gates.
|
||||
|
||||
The normal build/staging default is `auto`, shipping GPU support together with
|
||||
CPU fallback: Metal on macOS, Vulkan on Windows/Linux. Runtime detection chooses
|
||||
the usable device; users do not need hardware-specific builds. Release builders
|
||||
|
||||
+50
-1
@@ -5,7 +5,56 @@ inference, FFmpeg decoding and WebView tests use CPU only. UI launches set
|
||||
`WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--disable-gpu --disable-accelerated-video-decode'`
|
||||
and `SEARCHI_FORCE_CPU=1`.
|
||||
|
||||
## Native Linux media checkpoint (2026-10-13)
|
||||
## Native Linux runtime and headless checkpoint (2026-10-13)
|
||||
|
||||
The normal native Linux x86-64 worker now builds with Vulkan plus shared CPU
|
||||
dispatch. All 19 packaged libraries (14 CPU modules, Vulkan and four core
|
||||
libraries) pass dependency/RUNPATH checks and hashed resource staging. Actual
|
||||
copies outside the build tree pass genuine Google code-query inference on
|
||||
`libggml-cpu-zen4.so` and on a baseline-only `libggml-cpu-x64.so` extraction.
|
||||
Both produce finite normalized 768-dimensional vectors and report CPU with no
|
||||
Vulkan module loaded. See [dispatch evidence](evidence/linux-worker-cpu-dispatch.json)
|
||||
and [baseline evidence](evidence/linux-worker-cpu-baseline.json). These establish
|
||||
startup, origin-relative packaging and CPU inference, not broad retrieval
|
||||
quality, text/media reference parity, older hardware or GPU behavior.
|
||||
|
||||
An actual loader-free chroot also passes with requested device `auto` and no
|
||||
forced-CPU override. Its private mount namespace contains only the verified
|
||||
runtime, declared C/C++ ABI libraries and a read-only pinned model; Vulkan
|
||||
loaders, driver/ICD configuration and GPU devices are absent. It returns a
|
||||
normalized Google vector on CPU and reports Vulkan not loaded. See
|
||||
[missing-loader evidence](evidence/linux-worker-missing-loader.json). This proves
|
||||
Linux missing-loader CPU fallback without GPU execution; actual device loss,
|
||||
GPU precision and driver recovery remain open. The reusable check verifies
|
||||
namespace separation before any mount changes; two guard regressions pass.
|
||||
|
||||
Native Linux builder tests pass 7/7; staging passes 25 with two Windows-only
|
||||
skips; archive regressions pass 15/15. Packaging regressions use inert ELF
|
||||
fixtures and controlled metadata, so actual extracted CLI deployment remains
|
||||
a distinct gate. The Linux release CLI build passed (25m43s cold compilation).
|
||||
Initial actual archive preflight detected its glibc `libmvec.so.1` dependency;
|
||||
Ubuntu identifies that component as part of `libc6`. Adding that exact declared
|
||||
ABI component preserves unknown-dependency rejection. Actual archive creation
|
||||
now passes. The resulting unsigned headless archive is 148,157,038 bytes,
|
||||
SHA-256 `1e5a07edd4d94b49d2550914163637eca84087c802df0bb59b68168b849544e5`.
|
||||
All 29 listed files and permissions pass extraction verification into a path
|
||||
containing spaces. Installed release CLI startup, mixed document/code/audio
|
||||
indexing, automatic decoder discovery, real German semantic retrieval, code
|
||||
line citations and strict extension filtering pass. See
|
||||
[archive evidence](evidence/linux-headless-archive.json). This is an unpublished
|
||||
development candidate, not an installer or clean-machine certification.
|
||||
|
||||
Five genuine Google integration tests pass on native Linux, serially with one
|
||||
inference thread. Code task/debug/offset checks pass; text reference cosines
|
||||
range from 0.99981764 to 0.99994252 and debug/normal parity is 1.00000001.
|
||||
Image budgets 280/140/70 have reference cosines 0.99993253, 0.99976889 and
|
||||
0.99914157; audio parity is 0.99979352. These meet the 0.999 reference threshold.
|
||||
They establish runtime equivalence on this CPU host, not broad retrieval
|
||||
quality or GPU parity. Two additional genuine Google service tests pass:
|
||||
restart/stale-change/deletion behavior and failed rebuild policy preservation
|
||||
with retry. Seven real-model tests passed in total. No GPU provider executed.
|
||||
Exact model/worker hashes, test counts and reference similarities are retained
|
||||
in [Google CPU parity evidence](evidence/linux-google-cpu-parity.json).
|
||||
|
||||
Pinned BtbN Linux x86-64 FFmpeg/FFprobe archive size and SHA-256 verification
|
||||
passed under Ubuntu 24.04 WSL. Both native tools report the pinned version;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"platform": "Ubuntu 24.04 x86_64 under WSL in the Windows VM",
|
||||
"cpu_only": true,
|
||||
"inference_threads": 1,
|
||||
"serial_test_workers": true,
|
||||
"worker_sha256": "8e8b1046ca3387c078ff1f9d93eb05aba18d150b3c47d606bf1d6646b16bbd1f",
|
||||
"backbone_sha256": "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135",
|
||||
"projector_sha256": "c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f",
|
||||
"passed_tests": {"code": 2, "text": 2, "media": 1, "service": 2},
|
||||
"reference_cosine_threshold": 0.999,
|
||||
"text_reference_cosines": {
|
||||
"query": 0.99991829,
|
||||
"query_bn": 0.99992166,
|
||||
"query_ar": 0.99994252,
|
||||
"document": 0.99981764,
|
||||
"document_long": 0.99990818
|
||||
},
|
||||
"debug_normal_cosine": 1.00000001,
|
||||
"media_reference_cosines": {
|
||||
"screen_280": 0.99993253,
|
||||
"photo_140": 0.99976889,
|
||||
"frame_70": 0.99914157,
|
||||
"speech": 0.99979352
|
||||
},
|
||||
"service_scope": ["restart and changed/deleted source suppression", "failed policy rebuild preserves live reader and retry"],
|
||||
"scope": "Runtime equivalence to pinned DigUp references and small real source lifecycle fixtures; not broad retrieval quality or performance benchmarks",
|
||||
"limitations": ["GPU untested", "older physical CPUs untested", "distribution ABI coverage untested", "native desktop untested"]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"platform": "linux-x86_64",
|
||||
"cpu_only": true,
|
||||
"archive_sha256": "1e5a07edd4d94b49d2550914163637eca84087c802df0bb59b68168b849544e5",
|
||||
"archive_bytes": 148157038,
|
||||
"verified_files": 29,
|
||||
"extraction_path_contains_spaces": true,
|
||||
"automatic_media_helper_discovery": "passed",
|
||||
"embedding_report": {
|
||||
"media_files": 1,
|
||||
"media_failures": 0,
|
||||
"failures": [],
|
||||
"passages": 2,
|
||||
"chunks": 3,
|
||||
"generation": 1,
|
||||
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1:google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1:image280-frame70-audio30s-overlap2-distributed240-scene120-v1"
|
||||
},
|
||||
"semantic_top_filename": "supplier.md",
|
||||
"code_source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"strict_filter": "passed",
|
||||
"command_elapsed_ms": {
|
||||
"help": 1444.2557980000856,
|
||||
"index": 897.0072200000914,
|
||||
"embed": 12126.549685999977,
|
||||
"semantic_search": 964.4444650000423,
|
||||
"code_search": 1260.825207000039,
|
||||
"strict_filter": 966.029466000009
|
||||
},
|
||||
"timing_scope": "single deployment smoke samples; not retrieval/performance benchmarks",
|
||||
"limitations": [
|
||||
"GPU untested",
|
||||
"desktop untested",
|
||||
"clean-machine/ABI coverage untested",
|
||||
"broad retrieval quality untested"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"platform": "linux",
|
||||
"cpu_only": true,
|
||||
"baseline_only_extraction": true,
|
||||
"worker_sha256": "8e8b1046ca3387c078ff1f9d93eb05aba18d150b3c47d606bf1d6646b16bbd1f",
|
||||
"model_sha256": "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135",
|
||||
"files": [
|
||||
"libggml-base.so.0",
|
||||
"libggml-cpu-x64.so",
|
||||
"libggml-vulkan.so",
|
||||
"libggml.so.0",
|
||||
"libllama.so.0",
|
||||
"libmtmd.so.0",
|
||||
"runtime-libraries.json",
|
||||
"searchi-gemma-worker"
|
||||
],
|
||||
"hello": {
|
||||
"code_query_supported": true,
|
||||
"cpu_backend_module": "libggml-cpu-x64.so",
|
||||
"device": "cpu",
|
||||
"dimensions": 768,
|
||||
"media_supported": true,
|
||||
"model": "google/embeddinggemma-2",
|
||||
"ready": true,
|
||||
"runtime": "b11461-no-logits-v1",
|
||||
"threads": 1,
|
||||
"vulkan_module_loaded": false
|
||||
},
|
||||
"dimensions": 768,
|
||||
"norm": 1.0000000026410276,
|
||||
"startup_query_exit_ms": 1416.2597190002089,
|
||||
"timing_scope": "single diagnostic smoke sample; not a performance benchmark",
|
||||
"worker_dynamic_section": "\nDynamic section at offset 0x2f940 contains 35 entries:\n Tag Type Name/Value\n 0x0000000000000001 (NEEDED) Shared library: [libmtmd.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libllama.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libggml.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libggml-base.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libstdc++.so.6]\n 0x0000000000000001 (NEEDED) Shared library: [libm.so.6]\n 0x0000000000000001 (NEEDED) Shared library: [libgcc_s.so.1]\n 0x0000000000000001 (NEEDED) Shared library: [libc.so.6]\n 0x000000000000001d (RUNPATH) Library runpath: [$ORIGIN]\n 0x000000000000000c (INIT) 0x5000\n 0x000000000000000d (FINI) 0x27908\n 0x0000000000000019 (INIT_ARRAY) 0x306c8\n 0x000000000000001b (INIT_ARRAYSZ) 8 (bytes)\n 0x000000000000001a (FINI_ARRAY) 0x306d0\n 0x000000000000001c (FINI_ARRAYSZ) 8 (bytes)\n 0x000000006ffffef5 (GNU_HASH) 0x3b0\n 0x0000000000000005 (STRTAB) 0x14b0\n 0x0000000000000006 (SYMTAB) 0x4a8\n 0x000000000000000a (STRSZ) 5653 (bytes)\n 0x000000000000000b (SYMENT) 24 (bytes)\n 0x0000000000000015 (DEBUG) 0x0\n 0x0000000000000003 (PLTGOT) 0x30bb0\n 0x0000000000000002 (PLTRELSZ) 3072 (bytes)\n 0x0000000000000014 (PLTREL) RELA\n 0x0000000000000017 (JMPREL) 0x3598\n 0x0000000000000007 (RELA) 0x2da0\n 0x0000000000000008 (RELASZ) 2040 (bytes)\n 0x0000000000000009 (RELAENT) 24 (bytes)\n 0x000000000000001e (FLAGS) BIND_NOW\n 0x000000006ffffffb (FLAGS_1) Flags: NOW PIE\n 0x000000006ffffffe (VERNEED) 0x2c20\n 0x000000006fffffff (VERNEEDNUM) 4\n 0x000000006ffffff0 (VERSYM) 0x2ac6\n 0x000000006ffffff9 (RELACOUNT) 56\n 0x0000000000000000 (NULL) 0x0\n",
|
||||
"limitations": [
|
||||
"GPU untested",
|
||||
"older physical CPUs untested",
|
||||
"desktop and installer untested",
|
||||
"system-loader absence not physically exercised"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"platform": "linux",
|
||||
"cpu_only": true,
|
||||
"baseline_only_extraction": false,
|
||||
"worker_sha256": "8e8b1046ca3387c078ff1f9d93eb05aba18d150b3c47d606bf1d6646b16bbd1f",
|
||||
"model_sha256": "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135",
|
||||
"files": [
|
||||
"libggml-base.so.0",
|
||||
"libggml-cpu-alderlake.so",
|
||||
"libggml-cpu-cannonlake.so",
|
||||
"libggml-cpu-cascadelake.so",
|
||||
"libggml-cpu-cooperlake.so",
|
||||
"libggml-cpu-haswell.so",
|
||||
"libggml-cpu-icelake.so",
|
||||
"libggml-cpu-ivybridge.so",
|
||||
"libggml-cpu-piledriver.so",
|
||||
"libggml-cpu-sandybridge.so",
|
||||
"libggml-cpu-sapphirerapids.so",
|
||||
"libggml-cpu-skylakex.so",
|
||||
"libggml-cpu-sse42.so",
|
||||
"libggml-cpu-x64.so",
|
||||
"libggml-cpu-zen4.so",
|
||||
"libggml-vulkan.so",
|
||||
"libggml.so.0",
|
||||
"libllama.so.0",
|
||||
"libmtmd.so.0",
|
||||
"runtime-libraries.json",
|
||||
"searchi-gemma-worker"
|
||||
],
|
||||
"hello": {
|
||||
"code_query_supported": true,
|
||||
"cpu_backend_module": "libggml-cpu-zen4.so",
|
||||
"device": "cpu",
|
||||
"dimensions": 768,
|
||||
"media_supported": true,
|
||||
"model": "google/embeddinggemma-2",
|
||||
"ready": true,
|
||||
"runtime": "b11461-no-logits-v1",
|
||||
"threads": 1,
|
||||
"vulkan_module_loaded": false
|
||||
},
|
||||
"dimensions": 768,
|
||||
"norm": 1.0000000005534304,
|
||||
"startup_query_exit_ms": 2679.169430999991,
|
||||
"timing_scope": "single diagnostic smoke sample; not a performance benchmark",
|
||||
"worker_dynamic_section": "\nDynamic section at offset 0x2f940 contains 35 entries:\n Tag Type Name/Value\n 0x0000000000000001 (NEEDED) Shared library: [libmtmd.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libllama.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libggml.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libggml-base.so.0]\n 0x0000000000000001 (NEEDED) Shared library: [libstdc++.so.6]\n 0x0000000000000001 (NEEDED) Shared library: [libm.so.6]\n 0x0000000000000001 (NEEDED) Shared library: [libgcc_s.so.1]\n 0x0000000000000001 (NEEDED) Shared library: [libc.so.6]\n 0x000000000000001d (RUNPATH) Library runpath: [$ORIGIN]\n 0x000000000000000c (INIT) 0x5000\n 0x000000000000000d (FINI) 0x27908\n 0x0000000000000019 (INIT_ARRAY) 0x306c8\n 0x000000000000001b (INIT_ARRAYSZ) 8 (bytes)\n 0x000000000000001a (FINI_ARRAY) 0x306d0\n 0x000000000000001c (FINI_ARRAYSZ) 8 (bytes)\n 0x000000006ffffef5 (GNU_HASH) 0x3b0\n 0x0000000000000005 (STRTAB) 0x14b0\n 0x0000000000000006 (SYMTAB) 0x4a8\n 0x000000000000000a (STRSZ) 5653 (bytes)\n 0x000000000000000b (SYMENT) 24 (bytes)\n 0x0000000000000015 (DEBUG) 0x0\n 0x0000000000000003 (PLTGOT) 0x30bb0\n 0x0000000000000002 (PLTRELSZ) 3072 (bytes)\n 0x0000000000000014 (PLTREL) RELA\n 0x0000000000000017 (JMPREL) 0x3598\n 0x0000000000000007 (RELA) 0x2da0\n 0x0000000000000008 (RELASZ) 2040 (bytes)\n 0x0000000000000009 (RELAENT) 24 (bytes)\n 0x000000000000001e (FLAGS) BIND_NOW\n 0x000000006ffffffb (FLAGS_1) Flags: NOW PIE\n 0x000000006ffffffe (VERNEED) 0x2c20\n 0x000000006fffffff (VERNEEDNUM) 4\n 0x000000006ffffff0 (VERSYM) 0x2ac6\n 0x000000006ffffff9 (RELACOUNT) 56\n 0x0000000000000000 (NULL) 0x0\n",
|
||||
"limitations": [
|
||||
"GPU untested",
|
||||
"older physical CPUs untested",
|
||||
"desktop and installer untested",
|
||||
"system-loader absence not physically exercised"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"platform": "linux",
|
||||
"device_requested": "auto",
|
||||
"forced_cpu": false,
|
||||
"gpu_execution": false,
|
||||
"vulkan_loader_present": false,
|
||||
"seal": {
|
||||
"system_abi": [
|
||||
"libc.so.6",
|
||||
"libm.so.6",
|
||||
"libdl.so.2",
|
||||
"libpthread.so.0",
|
||||
"librt.so.1",
|
||||
"libstdc++.so.6",
|
||||
"libgcc_s.so.1"
|
||||
],
|
||||
"proc_read_only": true,
|
||||
"model_read_only": true,
|
||||
"private_mount_namespace_verified": true,
|
||||
"gpu_devices": false,
|
||||
"icd_manifests": false,
|
||||
"environment": {
|
||||
"PATH": "/usr/bin:/bin",
|
||||
"LANG": "C.UTF-8"
|
||||
}
|
||||
},
|
||||
"worker_sha256": "8e8b1046ca3387c078ff1f9d93eb05aba18d150b3c47d606bf1d6646b16bbd1f",
|
||||
"model_sha256": "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135",
|
||||
"hello": {
|
||||
"code_query_supported": true,
|
||||
"cpu_backend_module": "libggml-cpu-zen4.so",
|
||||
"device": "cpu",
|
||||
"dimensions": 768,
|
||||
"media_supported": true,
|
||||
"model": "google/embeddinggemma-2",
|
||||
"ready": true,
|
||||
"runtime": "b11461-no-logits-v1",
|
||||
"threads": 1,
|
||||
"vulkan_module_loaded": false
|
||||
},
|
||||
"dimensions": 768,
|
||||
"norm": 1.0000000005534304,
|
||||
"limitations": [
|
||||
"Actual GPU and driver recovery untested",
|
||||
"Distribution ABI compatibility untested"
|
||||
]
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
@@ -17,6 +18,87 @@ WINDOWS_CRT_FILES = (
|
||||
"msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll",
|
||||
"vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll",
|
||||
)
|
||||
LINUX_CPU_VARIANTS = {"x64", "sse42", "sandybridge", "ivybridge", "piledriver", "haswell",
|
||||
"skylakex", "cannonlake", "cascadelake", "icelake", "cooperlake", "zen4",
|
||||
"alderlake", "sapphirerapids"}
|
||||
LINUX_SYSTEM_LIBRARIES = {"libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0",
|
||||
"librt.so.1", "libstdc++.so.6", "libgcc_s.so.1", "ld-linux-x86-64.so.2"}
|
||||
|
||||
|
||||
def linux_dispatch(backend):
|
||||
"""Only normal Linux Vulkan modes use the packaged shared-module layout."""
|
||||
return (sys.platform.startswith("linux") and platform.machine().lower() in {"amd64", "x86_64"}
|
||||
and backend in {"auto", "vulkan"})
|
||||
|
||||
|
||||
def elf_dynamic(path):
|
||||
"""Read ELF metadata without executing the artifact or following ldd hooks."""
|
||||
output = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True)
|
||||
values = {}
|
||||
for key, value in re.findall(r"\((NEEDED|SONAME|RUNPATH|RPATH)\).*?\[([^\]]+)\]", output):
|
||||
values.setdefault(key, []).append(value)
|
||||
return values
|
||||
|
||||
|
||||
def bundle_linux_libraries(build):
|
||||
"""Flatten approved SONAME files beside the worker and record dependency closure.
|
||||
|
||||
System libc/C++ ABI remains a declared distro compatibility gate. Vulkan's
|
||||
loader is optional: only the dynamically loaded GPU module may depend on it.
|
||||
Preflight precedes flattening; the preceding CMake build itself is not a
|
||||
transactional release replacement. Staging performs that separate swap.
|
||||
"""
|
||||
build = Path(build)
|
||||
worker = build / "searchi-gemma-worker"
|
||||
candidates = [build / "bin" / f"lib{name}.so" for name in ("ggml", "ggml-base", "llama", "mtmd")]
|
||||
candidates += [build / f"libggml-cpu-{variant}.so" for variant in sorted(LINUX_CPU_VARIANTS)]
|
||||
candidates.append(build / "libggml-vulkan.so")
|
||||
records = []
|
||||
names = set()
|
||||
for link in candidates:
|
||||
if not link.is_file():
|
||||
raise RuntimeError(f"Required Linux runtime library is missing: {link.name}")
|
||||
source = link.resolve()
|
||||
if not source.is_relative_to(build.resolve()) or not source.is_file():
|
||||
raise RuntimeError("Linux runtime library escapes its build directory")
|
||||
metadata = elf_dynamic(source)
|
||||
sonames = metadata.get("SONAME", [])
|
||||
module = link.parent == build
|
||||
if ((not module and len(sonames) != 1) or len(sonames) > 1
|
||||
or any(not re.fullmatch(r"lib[A-Za-z0-9_-]+\.so(?:\.\d+)*", name) for name in sonames)):
|
||||
raise RuntimeError("Linux runtime has invalid or missing SONAME")
|
||||
# Backend discovery uses the unversioned module filename; core linkage
|
||||
# uses its SONAME. Both destinations are actual files, never symlinks.
|
||||
name = link.name if module else sonames[0]
|
||||
if name in names:
|
||||
raise RuntimeError("Duplicate Linux runtime library")
|
||||
names.add(name)
|
||||
destination = build / name
|
||||
records.append((source, destination, metadata))
|
||||
all_names = names
|
||||
for path, metadata in [(worker, elf_dynamic(worker)), *[(destination, metadata) for _, destination, metadata in records]]:
|
||||
if metadata.get("RPATH") or metadata.get("RUNPATH") != ["$ORIGIN"]:
|
||||
raise RuntimeError(f"Linux runtime requires origin-only RUNPATH: {path.name}")
|
||||
needed = set(metadata.get("NEEDED", []))
|
||||
allowed = all_names | LINUX_SYSTEM_LIBRARIES
|
||||
if path.name == "libggml-vulkan.so":
|
||||
allowed = allowed | {"libvulkan.so.1"}
|
||||
if needed - allowed:
|
||||
raise RuntimeError(f"Unpackaged Linux runtime dependency for {path.name}: {sorted(needed - allowed)}")
|
||||
for source, destination, _ in records:
|
||||
if destination != source:
|
||||
if destination.is_symlink():
|
||||
destination.unlink()
|
||||
shutil.copy2(source, destination)
|
||||
def checksum(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
inventory = {"platform": "linux", "worker_sha256": checksum(worker),
|
||||
"libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)}
|
||||
for _, path, _ in sorted(records)]}
|
||||
temporary = build / "runtime-libraries.json.partial"
|
||||
temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8")
|
||||
os.replace(temporary, build / "runtime-libraries.json")
|
||||
|
||||
|
||||
def bundle_windows_crt(build):
|
||||
@@ -90,7 +172,7 @@ def main():
|
||||
reverse = subprocess.run(["git", "apply", "--reverse", "--check", str(patch)], cwd=source, capture_output=True)
|
||||
if reverse.returncode:
|
||||
raise RuntimeError("Runtime memory patch cannot be applied or verified")
|
||||
if os.name == "nt" and args.backend != "cpu":
|
||||
if (os.name == "nt" and args.backend != "cpu") or linux_dispatch(args.backend):
|
||||
for patch_name in ["cpu-dispatch.patch", "cpu-only-loader.patch"]:
|
||||
dispatch_patch = Path(__file__).with_name(patch_name)
|
||||
check = subprocess.run(["git", "apply", "--check", str(dispatch_patch)], cwd=source, capture_output=True)
|
||||
@@ -125,6 +207,8 @@ def main():
|
||||
temporary = directory / "runtime-libraries.json.partial"
|
||||
temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8")
|
||||
os.replace(temporary, directory / "runtime-libraries.json")
|
||||
if linux_dispatch(args.backend):
|
||||
bundle_linux_libraries(build)
|
||||
print(f"Runtime build complete: {build}")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Prove Linux auto CPU fallback with the Vulkan loader physically absent.
|
||||
|
||||
Run as root on x86_64 Linux with util-linux unshare/mount/umount and chroot.
|
||||
This tool never escalates privileges. It launches itself inside a private mount
|
||||
namespace and verifies that namespace before changing mount propagation. The
|
||||
sealed chroot includes approved runtime files, standard C/C++ ABI libraries and
|
||||
a read-only Google model, without Vulkan loaders, ICDs, drivers or GPU devices.
|
||||
This tests missing-loader fallback, not GPU execution or distro compatibility.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ABI = ("libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0",
|
||||
"librt.so.1", "libstdc++.so.6", "libgcc_s.so.1")
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def checksum(path):
|
||||
with Path(path).open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def namespace(pid="self"):
|
||||
return Path(f"/proc/{pid}/ns/mnt").stat().st_ino
|
||||
|
||||
|
||||
def isolated_check(args, commands):
|
||||
# Check the still-live outer process as well as its recorded namespace.
|
||||
# A direct child invocation in the outer namespace fails before any mount.
|
||||
outer = namespace(args.outer_pid)
|
||||
require(outer == args.outer_namespace and namespace() != outer,
|
||||
"Private mount namespace was not established; refusing mount changes")
|
||||
subprocess.run([commands["mount"], "--make-rprivate", "/"], check=True)
|
||||
from stage_runtime import verify_runtime
|
||||
worker = args.worker.resolve()
|
||||
dependencies = verify_runtime(worker, True)
|
||||
manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||
artifact = manifest["files"][0]
|
||||
model = args.models.resolve() / artifact["path"]
|
||||
require(model.is_file() and model.stat().st_size == artifact["bytes"]
|
||||
and checksum(model) == artifact["sha256"], "Google model verification failed")
|
||||
base = Path(tempfile.gettempdir()).resolve()
|
||||
root = Path(tempfile.mkdtemp(prefix="searchi-loader-chroot-", dir=base)).resolve()
|
||||
require(root.parent == base and root.stat().st_uid == 0,
|
||||
"Invalid owned chroot directory")
|
||||
mounted = []
|
||||
try:
|
||||
app = root / "app"
|
||||
app.mkdir()
|
||||
for source in [worker, *dependencies]:
|
||||
destination = app / source.name
|
||||
shutil.copy2(source, destination)
|
||||
require(checksum(source) == checksum(destination), "Copied runtime checksum mismatch")
|
||||
abi_dir = root / "lib/x86_64-linux-gnu"
|
||||
abi_dir.mkdir(parents=True)
|
||||
for name in ABI:
|
||||
shutil.copy2(Path("/lib/x86_64-linux-gnu") / name, abi_dir / name)
|
||||
(root / "lib64").mkdir()
|
||||
shutil.copy2("/lib64/ld-linux-x86-64.so.2", root / "lib64/ld-linux-x86-64.so.2")
|
||||
(root / "models").mkdir()
|
||||
mounted_model = root / "models/model.gguf"
|
||||
mounted_model.touch()
|
||||
proc = root / "proc"
|
||||
proc.mkdir()
|
||||
subprocess.run([commands["mount"], "--bind", str(model), str(mounted_model)], check=True)
|
||||
mounted.append(mounted_model)
|
||||
subprocess.run([commands["mount"], "-o", "remount,bind,ro", str(mounted_model)], check=True)
|
||||
subprocess.run([commands["mount"], "-t", "proc", "-o", "ro,nosuid,nodev,noexec", "proc", str(proc)], check=True)
|
||||
mounted.append(proc)
|
||||
require(not (root / "dev").exists() and not (root / "etc").exists(),
|
||||
"Unexpected devices or driver configuration in chroot")
|
||||
require({path.name for path in abi_dir.iterdir()} == set(ABI), "Unexpected system library")
|
||||
require({path.name for path in app.iterdir()} == {worker.name, *(p.name for p in dependencies)},
|
||||
"Unexpected runtime file")
|
||||
# The worker runs with no inherited loader/ggml variables and no CPU
|
||||
# override: the absence of libvulkan must cause its actual auto fallback.
|
||||
environment = {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"}
|
||||
process = subprocess.Popen([commands["chroot"], str(root), "/app/searchi-gemma-worker",
|
||||
"--model", "/models/model.gguf", "--threads", "1", "--device", "auto"],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, encoding="utf-8", env=environment)
|
||||
try:
|
||||
stdout, stderr = process.communicate(json.dumps({"op": "code_query",
|
||||
"text": "retry failed requests with exponential backoff"}) + "\n", timeout=180)
|
||||
except BaseException:
|
||||
process.kill()
|
||||
process.wait(timeout=10)
|
||||
raise
|
||||
require(process.returncode == 0, f"Worker failed: {stderr[-4096:]}")
|
||||
lines = [json.loads(line) for line in stdout.splitlines()]
|
||||
require(len(lines) == 2, "Unexpected protocol output")
|
||||
hello, result = lines
|
||||
require(hello.get("ready") and hello.get("device") == "cpu"
|
||||
and not hello.get("vulkan_module_loaded") and result.get("ok")
|
||||
and result.get("device") == "cpu", "Expected auto fallback to CPU without Vulkan loading")
|
||||
vectors = result.get("vectors", [])
|
||||
require(len(vectors) == 1 and len(vectors[0]) == 768
|
||||
and all(isinstance(v, (float, int)) and math.isfinite(v) for v in vectors[0]),
|
||||
"Invalid real Google vector")
|
||||
norm = math.sqrt(sum(value * value for value in vectors[0]))
|
||||
require(abs(norm - 1) < .001, "Google vector is not normalized")
|
||||
evidence = {"platform": "linux", "device_requested": "auto", "forced_cpu": False,
|
||||
"gpu_execution": False, "vulkan_loader_present": False,
|
||||
"seal": {"system_abi": list(ABI), "proc_read_only": True, "model_read_only": True,
|
||||
"private_mount_namespace_verified": True, "gpu_devices": False,
|
||||
"icd_manifests": False, "environment": environment},
|
||||
"worker_sha256": checksum(worker), "model_sha256": checksum(model),
|
||||
"hello": hello, "dimensions": len(vectors[0]), "norm": norm,
|
||||
"limitations": ["Actual GPU and driver recovery untested", "Distribution ABI compatibility untested"]}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"auto_without_loader": "cpu", "cpu_module": hello["cpu_backend_module"],
|
||||
"vulkan_loaded": False}))
|
||||
finally:
|
||||
failures = []
|
||||
for path in reversed(mounted):
|
||||
result = subprocess.run([commands["umount"], str(path)], check=False)
|
||||
if result.returncode:
|
||||
failures.append(str(path))
|
||||
# Never recursively remove a directory containing a live bind mount.
|
||||
require(not failures, f"Mount cleanup failed; preserved owned directory {root}")
|
||||
require(root.parent == base and root.name.startswith("searchi-loader-chroot-"),
|
||||
"Refusing cleanup outside owned temporary directory")
|
||||
shutil.rmtree(root)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--worker", type=Path, required=True)
|
||||
parser.add_argument("--models", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--outer-pid", type=int, help=argparse.SUPPRESS)
|
||||
parser.add_argument("--outer-namespace", type=int, help=argparse.SUPPRESS)
|
||||
args = parser.parse_args()
|
||||
require(sys.platform.startswith("linux") and platform.machine().lower() in {"amd64", "x86_64"},
|
||||
"Run this check on x86_64 Linux")
|
||||
require(os.geteuid() == 0, "Run explicitly as root; this tool does not escalate privileges")
|
||||
commands = {name: shutil.which(name) for name in ("unshare", "mount", "umount", "chroot")}
|
||||
require(all(commands.values()), "Install util-linux mount tools and chroot")
|
||||
if args.outer_pid is not None or args.outer_namespace is not None:
|
||||
require(args.outer_pid is not None and args.outer_namespace is not None,
|
||||
"Incomplete namespace handoff")
|
||||
isolated_check(args, commands)
|
||||
else:
|
||||
subprocess.run([commands["unshare"], "--mount", "--fork", sys.executable, str(Path(__file__).resolve()),
|
||||
"--worker", str(args.worker.resolve()), "--models", str(args.models.resolve()),
|
||||
"--output", str(args.output.resolve()), "--outer-pid", str(os.getpid()),
|
||||
"--outer-namespace", str(namespace())], check=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Exercise a real Linux distribution worker on CPU, outside its build directory.
|
||||
|
||||
This checks genuine Google inference and origin-relative runtime packaging.
|
||||
It does not certify GPU operation, older processors or a Linux desktop.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
def sha256(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--worker", type=Path, required=True)
|
||||
parser.add_argument("--models", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--baseline", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if os.name != "posix" or not Path("/proc/self/exe").exists():
|
||||
parser.error("Run this check natively on Linux")
|
||||
# Reuse exactly the staging verifier: hashes and bounded inventory names
|
||||
# must agree with what enters a distribution.
|
||||
from stage_runtime import verify_runtime
|
||||
dependencies = verify_runtime(args.worker.resolve(), True)
|
||||
manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
|
||||
model = args.models.resolve() / manifest["files"][0]["path"]
|
||||
if model.stat().st_size != manifest["files"][0]["bytes"] or sha256(model) != manifest["files"][0]["sha256"]:
|
||||
raise RuntimeError("Google model checksum mismatch")
|
||||
with tempfile.TemporaryDirectory(prefix="searchi-linux-smoke-") as temporary:
|
||||
target = Path(temporary)
|
||||
copied = []
|
||||
for source in [args.worker.resolve(), *dependencies]:
|
||||
# Only exact approved CPU module names are omitted. Core libraries
|
||||
# and GPU support remain present in the baseline extraction.
|
||||
if args.baseline and source.name.startswith("libggml-cpu-") and source.name != "libggml-cpu-x64.so":
|
||||
continue
|
||||
destination = target / source.name
|
||||
shutil.copy2(source, destination)
|
||||
if sha256(destination) != sha256(source):
|
||||
raise RuntimeError("Extracted runtime checksum mismatch")
|
||||
copied.append(source.name)
|
||||
worker = target / args.worker.name
|
||||
dynamic = subprocess.check_output(["readelf", "-d", str(worker)], text=True)
|
||||
if "libvulkan" in dynamic or "libggml-vulkan" in dynamic:
|
||||
raise RuntimeError("Worker requires Vulkan at process startup; CPU fallback cannot work without the loader")
|
||||
if "$ORIGIN" not in dynamic:
|
||||
raise RuntimeError("Worker lacks origin-relative runtime lookup")
|
||||
environment = dict(os.environ, SEARCHI_FORCE_CPU="1", GGML_DISABLE_VULKAN="1")
|
||||
environment.pop("LD_LIBRARY_PATH", None)
|
||||
environment.pop("GGML_BACKEND_PATH", None)
|
||||
started = time.perf_counter()
|
||||
process = subprocess.Popen([str(worker), "--model", str(model), "--threads", "1", "--device", "cpu"],
|
||||
cwd=target, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, encoding="utf-8", env=environment)
|
||||
try:
|
||||
stdout, stderr = process.communicate(json.dumps({"op": "code_query", "text": "retry failed requests with exponential backoff"}) + "\n", timeout=180)
|
||||
except BaseException:
|
||||
process.kill()
|
||||
process.wait(timeout=10)
|
||||
raise
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
if process.returncode:
|
||||
raise RuntimeError(f"Worker failed ({process.returncode}): {stderr[-4096:]}")
|
||||
lines = [json.loads(line) for line in stdout.splitlines()]
|
||||
if len(lines) != 2:
|
||||
raise RuntimeError("Unexpected worker protocol output")
|
||||
hello, result = lines
|
||||
if not hello.get("ready") or hello.get("device") != "cpu" or not result.get("ok") or result.get("device") != "cpu":
|
||||
raise RuntimeError("Expected genuine ready CPU inference")
|
||||
if hello.get("vulkan_module_loaded"):
|
||||
raise RuntimeError("CPU check loaded a Vulkan provider")
|
||||
vectors = result.get("vectors")
|
||||
vector = vectors[0] if isinstance(vectors, list) and len(vectors) == 1 else None
|
||||
if not isinstance(vector, list) or len(vector) != 768 or not all(isinstance(value, (int, float)) and math.isfinite(value) for value in vector):
|
||||
raise RuntimeError("Invalid Google embedding vector")
|
||||
norm = math.sqrt(sum(value * value for value in vector))
|
||||
if abs(norm - 1) > 0.001:
|
||||
raise RuntimeError("Google vector is not normalized")
|
||||
module = hello.get("cpu_backend_module", "")
|
||||
if args.baseline and module != "libggml-cpu-x64.so":
|
||||
raise RuntimeError(f"Baseline extraction selected unexpected CPU module: {module}")
|
||||
evidence = {"platform": "linux", "cpu_only": True, "baseline_only_extraction": args.baseline,
|
||||
"worker_sha256": sha256(args.worker), "model_sha256": sha256(model),
|
||||
"files": sorted(copied), "hello": hello, "dimensions": len(vector), "norm": norm,
|
||||
"startup_query_exit_ms": elapsed_ms,
|
||||
"timing_scope": "single diagnostic smoke sample; not a performance benchmark",
|
||||
"worker_dynamic_section": dynamic,
|
||||
"limitations": ["GPU untested", "older physical CPUs untested", "desktop and installer untested", "system-loader absence not physically exercised"]}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"ready": True, "cpu_module": module, "dimensions": 768, "baseline": args.baseline}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -15,6 +15,11 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
|
||||
|
||||
|
||||
def library_name(name):
|
||||
return isinstance(name, str) and bool(re.fullmatch(
|
||||
r"(?:[A-Za-z0-9_-]+\.dll|lib[A-Za-z0-9_-]+\.so(?:\.\d+)*)", name))
|
||||
|
||||
|
||||
def _is_link(path):
|
||||
"""Reject Windows reparse points, including junctions, on Python 3.11+."""
|
||||
path = Path(path)
|
||||
@@ -208,7 +213,7 @@ def _previous_owned(destination):
|
||||
raise RuntimeError("Invalid previous runtime inventory")
|
||||
for record in records:
|
||||
name = record.get("path") if isinstance(record, dict) else None
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+\.dll", name):
|
||||
if not library_name(name):
|
||||
raise RuntimeError("Invalid previous runtime library path")
|
||||
names.add(name)
|
||||
return names
|
||||
@@ -345,7 +350,7 @@ def verify_runtime(source, required=False):
|
||||
if not isinstance(record, dict):
|
||||
raise RuntimeError("Invalid runtime library metadata")
|
||||
name = record.get("path", "")
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+\.dll", name) or name.lower() in names:
|
||||
if not library_name(name) or name.lower() in names:
|
||||
raise RuntimeError("Invalid or duplicate runtime library path")
|
||||
names.add(name.lower())
|
||||
artifact = source.parent / name
|
||||
@@ -361,8 +366,17 @@ def verify_runtime(source, required=False):
|
||||
if hashlib.file_digest(stream, "sha256").hexdigest() != checksum:
|
||||
raise RuntimeError(f"Runtime library checksum mismatch: {name}")
|
||||
paths.append(artifact)
|
||||
if required and not ({"ggml-cpu-x64.dll", "ggml-vulkan.dll", "vulkan-1.dll", "ggml.dll", "ggml-base.dll", "llama.dll", "mtmd.dll"} | CRT_LIBRARIES).issubset(names):
|
||||
raise RuntimeError("Runtime inventory lacks required CPU/GPU libraries")
|
||||
if required:
|
||||
if (source.suffix == ".exe") == (manifest.get("platform") == "linux"):
|
||||
raise RuntimeError("Runtime inventory platform does not match worker")
|
||||
if manifest.get("platform") == "linux":
|
||||
core = {"ggml", "ggml-base", "llama", "mtmd"}
|
||||
present = {name for name in core if any(re.fullmatch(rf"lib{re.escape(name)}\.so(?:\.\d+)*", file) for file in names)}
|
||||
complete = present == core and {"libggml-cpu-x64.so", "libggml-vulkan.so"}.issubset(names)
|
||||
else:
|
||||
complete = ({"ggml-cpu-x64.dll", "ggml-vulkan.dll", "vulkan-1.dll", "ggml.dll", "ggml-base.dll", "llama.dll", "mtmd.dll"} | CRT_LIBRARIES).issubset(names)
|
||||
if not complete:
|
||||
raise RuntimeError("Runtime inventory lacks required CPU/GPU libraries")
|
||||
return [inventory, *paths]
|
||||
|
||||
|
||||
@@ -401,7 +415,8 @@ def main():
|
||||
license_path = ROOT / ".searchi/references/llama.cpp/LICENSE"
|
||||
if not license_path.is_file():
|
||||
raise RuntimeError("Native worker license is missing")
|
||||
dependencies = verify_runtime(source, os.name == "nt" and args.backend in {"auto", "vulkan"})
|
||||
required_inventory = (os.name == "nt" or os.sys.platform.startswith("linux")) and args.backend in {"auto", "vulkan"}
|
||||
dependencies = verify_runtime(source, required_inventory)
|
||||
crt_notice = source.parent / "MSVC-REDISTRIBUTION.txt"
|
||||
if os.name == "nt" and args.backend != "cpu":
|
||||
if not crt_notice.is_file():
|
||||
@@ -425,7 +440,7 @@ def main():
|
||||
sources.extend((path, f"embeddinggemma2/{path.name}") for path in artifacts)
|
||||
|
||||
def validate(candidate):
|
||||
verify_runtime(candidate / source.name, os.name == "nt" and args.backend in {"auto", "vulkan"})
|
||||
verify_runtime(candidate / source.name, required_inventory)
|
||||
if model_manifest:
|
||||
for artifact, spec in zip(artifacts, model_manifest["files"], strict=True):
|
||||
if _record(candidate / "embeddinggemma2" / artifact.name) != (spec["bytes"], spec["sha256"]):
|
||||
|
||||
@@ -1,11 +1,87 @@
|
||||
"""Redistributable preparation tests; no native builds or GPU execution."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import build_runtime
|
||||
from build_runtime import bundle_windows_crt, WINDOWS_CRT_FILES
|
||||
|
||||
|
||||
class LinuxRuntimeTests(unittest.TestCase):
|
||||
def test_only_normal_linux_vulkan_modes_use_shared_dispatch(self):
|
||||
with patch.object(build_runtime.sys, "platform", "linux"), patch.object(build_runtime.platform, "machine", return_value="x86_64"):
|
||||
for backend in ["auto", "vulkan"]:
|
||||
self.assertTrue(build_runtime.linux_dispatch(backend))
|
||||
for backend in ["cpu", "cuda", "metal"]:
|
||||
self.assertFalse(build_runtime.linux_dispatch(backend))
|
||||
with patch.object(build_runtime.sys, "platform", "darwin"):
|
||||
self.assertFalse(build_runtime.linux_dispatch("auto"))
|
||||
with patch.object(build_runtime.sys, "platform", "linux"), patch.object(build_runtime.platform, "machine", return_value="aarch64"):
|
||||
self.assertFalse(build_runtime.linux_dispatch("auto"))
|
||||
self.assertFalse(build_runtime.linux_dispatch("vulkan"))
|
||||
|
||||
def test_readelf_parses_needed_soname_and_runpath_without_executing_worker(self):
|
||||
output = " 0x1 (NEEDED) Shared library: [libggml.so.0]\n 0xe (SONAME) Library soname: [libggml-cpu-x64.so]\n 0x1d (RUNPATH) Library runpath: [$ORIGIN]\n"
|
||||
with patch.object(build_runtime.subprocess, "check_output", return_value=output) as command:
|
||||
result = build_runtime.elf_dynamic(Path("worker"))
|
||||
self.assertEqual(result, {"NEEDED": ["libggml.so.0"], "SONAME": ["libggml-cpu-x64.so"], "RUNPATH": ["$ORIGIN"]})
|
||||
command.assert_called_once_with(["readelf", "--dynamic", "worker"], text=True)
|
||||
|
||||
def fixture(self, root):
|
||||
root = Path(root)
|
||||
(root / "bin").mkdir()
|
||||
(root / "searchi-gemma-worker").write_bytes(b"worker")
|
||||
for name in ["ggml", "ggml-base", "llama", "mtmd"]:
|
||||
(root / "bin" / f"lib{name}.so").write_bytes(name.encode())
|
||||
for variant in build_runtime.LINUX_CPU_VARIANTS:
|
||||
(root / f"libggml-cpu-{variant}.so").write_bytes(variant.encode())
|
||||
(root / "libggml-vulkan.so").write_bytes(b"vulkan")
|
||||
return root
|
||||
|
||||
def metadata(self, path):
|
||||
result = {"RUNPATH": ["$ORIGIN"], "NEEDED": ["libstdc++.so.6", "libc.so.6"]}
|
||||
if path.parent.name == "bin":
|
||||
result["SONAME"] = [path.name + ".0"]
|
||||
if path.name == "libggml-vulkan.so":
|
||||
result["NEEDED"].append("libvulkan.so.1")
|
||||
return result
|
||||
|
||||
def test_flattened_core_and_bounded_modules_have_hashed_inventory(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = self.fixture(directory)
|
||||
with patch.object(build_runtime, "elf_dynamic", side_effect=self.metadata):
|
||||
build_runtime.bundle_linux_libraries(root)
|
||||
manifest = json.loads((root / "runtime-libraries.json").read_text())
|
||||
self.assertEqual(manifest["platform"], "linux")
|
||||
self.assertEqual(len(manifest["libraries"]), 5 + len(build_runtime.LINUX_CPU_VARIANTS))
|
||||
self.assertTrue((root / "libllama.so.0").is_file())
|
||||
self.assertTrue(all(not (root / item["path"]).is_symlink() for item in manifest["libraries"]))
|
||||
self.assertTrue(all(len(item["sha256"]) == 64 for item in manifest["libraries"]))
|
||||
|
||||
def test_startup_gpu_dependency_and_build_tree_rpath_are_rejected(self):
|
||||
for problem in ["gpu", "rpath", "missing"]:
|
||||
with self.subTest(problem=problem), tempfile.TemporaryDirectory() as directory:
|
||||
root = self.fixture(directory)
|
||||
previous = root / "libllama.so.0"
|
||||
previous.write_bytes(b"previous flattened core")
|
||||
def metadata(path):
|
||||
result = self.metadata(path)
|
||||
if path.name == "searchi-gemma-worker":
|
||||
if problem == "gpu":
|
||||
result["NEEDED"].append("libvulkan.so.1")
|
||||
elif problem == "rpath":
|
||||
result["RUNPATH"] = ["/private/build"]
|
||||
else:
|
||||
result["NEEDED"].append("libunpackaged.so.9")
|
||||
return result
|
||||
with patch.object(build_runtime, "elf_dynamic", side_effect=metadata), self.assertRaises(RuntimeError):
|
||||
build_runtime.bundle_linux_libraries(root)
|
||||
self.assertFalse((root / "runtime-libraries.json").exists())
|
||||
self.assertEqual(previous.read_bytes(), b"previous flattened core")
|
||||
|
||||
|
||||
class CrtTests(unittest.TestCase):
|
||||
def fixture(self, directory):
|
||||
root = Path(directory)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Namespace guards must reject unsafe handoffs before any mount operation."""
|
||||
import argparse
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import linux_loader_smoke
|
||||
|
||||
|
||||
class NamespaceGuardTests(unittest.TestCase):
|
||||
def test_same_namespace_refuses_mount_before_runtime_access(self):
|
||||
args = argparse.Namespace(outer_pid=123, outer_namespace=42)
|
||||
with patch.object(linux_loader_smoke, "namespace", return_value=42), \
|
||||
patch.object(linux_loader_smoke.subprocess, "run") as run:
|
||||
with self.assertRaisesRegex(RuntimeError, "Private mount namespace"):
|
||||
linux_loader_smoke.isolated_check(args, {"mount": "mount"})
|
||||
run.assert_not_called()
|
||||
|
||||
def test_changed_outer_namespace_refuses_mount(self):
|
||||
args = argparse.Namespace(outer_pid=123, outer_namespace=42)
|
||||
with patch.object(linux_loader_smoke, "namespace", return_value=43), \
|
||||
patch.object(linux_loader_smoke.subprocess, "run") as run:
|
||||
with self.assertRaisesRegex(RuntimeError, "Private mount namespace"):
|
||||
linux_loader_smoke.isolated_check(args, {"mount": "mount"})
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -12,6 +12,44 @@ import stage_runtime
|
||||
|
||||
|
||||
class StagingTests(unittest.TestCase):
|
||||
def test_linux_inventory_stages_core_and_modules_and_removes_previous_owned_library(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
runtime = root / "build"
|
||||
runtime.mkdir()
|
||||
worker = runtime / "searchi-gemma-worker"
|
||||
worker.write_bytes(b"ELF fixture for integrity, not executable inference")
|
||||
names = ["libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0",
|
||||
"libggml-cpu-x64.so", "libggml-vulkan.so"]
|
||||
records = []
|
||||
for name in names:
|
||||
path = runtime / name
|
||||
path.write_bytes(name.encode())
|
||||
records.append({"path": name, "bytes": path.stat().st_size,
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest()})
|
||||
inventory = runtime / "runtime-libraries.json"
|
||||
inventory.write_text(json.dumps({"platform": "linux", "worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records}))
|
||||
dependencies = stage_runtime.verify_runtime(worker, True)
|
||||
destination = root / "resources"
|
||||
destination.mkdir()
|
||||
(destination / "libggml-cpu-old.so").write_bytes(b"old owned backend")
|
||||
(destination / "runtime-libraries.json").write_text(json.dumps({"libraries": [{"path": "libggml-cpu-old.so"}]}))
|
||||
(destination / "note.txt").write_bytes(b"unrelated")
|
||||
stage_runtime.stage_resources(destination, [(worker, worker.name), *[(path, path.name) for path in dependencies]],
|
||||
lambda candidate: stage_runtime.verify_runtime(candidate / worker.name, True))
|
||||
self.assertFalse((destination / "libggml-cpu-old.so").exists())
|
||||
self.assertEqual((destination / "note.txt").read_bytes(), b"unrelated")
|
||||
self.assertEqual(len(stage_runtime.verify_runtime(destination / worker.name, True)), 7)
|
||||
(destination / "libggml-vulkan.so").write_bytes(b"corrupt")
|
||||
with self.assertRaises(RuntimeError):
|
||||
stage_runtime.verify_runtime(destination / worker.name, True)
|
||||
|
||||
def test_runtime_library_names_reject_paths_and_unbounded_suffixes(self):
|
||||
for name in ["../libggml.so", "libggml.so.x", "libggml.so/1", "libggml.so..1", "ggml.dylib"]:
|
||||
with self.subTest(name=name):
|
||||
self.assertFalse(stage_runtime.library_name(name))
|
||||
self.assertTrue(stage_runtime.library_name("libggml-base.so.0.13.0"))
|
||||
|
||||
def resources(self, directory):
|
||||
root = Path(directory)
|
||||
destination = root / "src-tauri/resources"
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Check a real extracted Linux headless candidate using Google CPU inference.
|
||||
|
||||
This is an installed-layout smoke test, not a broad relevance benchmark or
|
||||
clean-machine certification. No GPU provider or network setup is exercised.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_FILE = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def sha256(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def extract_checked(archive_path, destination):
|
||||
from package_linux import safe_name
|
||||
with tarfile.open(archive_path, "r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
if not 1 <= len(members) <= 128 or len({item.name for item in members}) != len(members):
|
||||
raise ValueError("Invalid candidate member inventory")
|
||||
for member in members:
|
||||
safe_name(member.name)
|
||||
if not member.isfile() or not 0 < member.size <= MAX_FILE:
|
||||
raise ValueError("Unexpected candidate member")
|
||||
manifest_member = archive.getmember("PACKAGE.json")
|
||||
if manifest_member.size > 65536:
|
||||
raise ValueError("Oversized candidate manifest")
|
||||
manifest = json.load(archive.extractfile(manifest_member))
|
||||
records = manifest["files"]
|
||||
expected = {record["path"]: record for record in records}
|
||||
if (manifest.get("platform") != "linux" or manifest.get("architecture") != "x86_64"
|
||||
or len(expected) != len(records) or set(expected) != {member.name for member in members} - {"PACKAGE.json"}
|
||||
or sum(member.size for member in members) > 1024 * 1024 * 1024):
|
||||
raise ValueError("Candidate inventory does not cover archive")
|
||||
for member in members:
|
||||
target = destination / member.name
|
||||
if not target.resolve().is_relative_to(destination.resolve()):
|
||||
raise ValueError("Candidate path escapes extraction")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with archive.extractfile(member) as source, target.open("xb") as output:
|
||||
shutil.copyfileobj(source, output, length=65536)
|
||||
if member.name != "PACKAGE.json":
|
||||
record = expected[member.name]
|
||||
mode = int(record["mode"], 8)
|
||||
if (target.stat().st_size != record["bytes"] or sha256(target) != record["sha256"]
|
||||
or mode not in {0o644, 0o755} or member.mode != mode):
|
||||
raise ValueError("Candidate bytes or permissions mismatch")
|
||||
target.chmod(mode)
|
||||
return manifest
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--archive", type=Path, required=True)
|
||||
parser.add_argument("--models", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if os.name != "posix" or not Path("/proc/self/exe").exists():
|
||||
parser.error("Run natively on Linux")
|
||||
models = args.models.resolve()
|
||||
environment = dict(os.environ, SEARCHI_FORCE_CPU="1", GGML_DISABLE_VULKAN="1")
|
||||
environment.pop("LD_LIBRARY_PATH", None)
|
||||
environment.pop("GGML_BACKEND_PATH", None)
|
||||
environment.pop("SEARCHI_FFMPEG_PATH", None)
|
||||
environment.pop("SEARCHI_FFPROBE_PATH", None)
|
||||
timings = {}
|
||||
with tempfile.TemporaryDirectory(prefix="searchi archive QA ") as directory:
|
||||
root = Path(directory)
|
||||
installed = root / "installed"
|
||||
installed.mkdir()
|
||||
manifest = extract_checked(args.archive, installed)
|
||||
cli, worker = installed / "searchi", installed / "searchi-gemma-worker"
|
||||
corpus = root / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "supplier.md").write_text("The supplier partnership may be ended with sixty days written notice.\n", encoding="utf-8")
|
||||
(corpus / "retry.py").write_text("import time\n\ndef fetch_with_backoff(operation):\n delay = 1\n for attempt in range(5):\n try:\n return operation()\n except Exception:\n time.sleep(delay)\n delay *= 2\n", encoding="utf-8")
|
||||
shutil.copy2(ROOT / "crates/searchi-semantic/tests/fixtures/digup-speech.wav", corpus / "speech.wav")
|
||||
db = root / "index.sqlite"
|
||||
|
||||
def run(label, arguments, json_output=False):
|
||||
started = time.perf_counter()
|
||||
output = subprocess.run([str(cli), *map(str, arguments)], cwd=installed, env=environment,
|
||||
check=True, capture_output=True, text=True, encoding="utf-8", timeout=180)
|
||||
timings[label] = (time.perf_counter() - started) * 1000
|
||||
return json.loads(output.stdout) if json_output else output.stdout
|
||||
|
||||
run("help", ["--help"])
|
||||
run("index", ["index", "--db", db, "--root", corpus])
|
||||
# Omit explicit decoder paths to exercise installed helper discovery.
|
||||
report = run("embed", ["embed", "--db", db, "--model-dir", models,
|
||||
"--embedding-worker", worker, "--cpu-threads", "1"], True)
|
||||
if report["media_files"] != 1 or report["media_failures"] != 0 or report["passages"] != 2 or report["chunks"] < 3:
|
||||
raise RuntimeError("Installed mixed document/code/audio indexing failed")
|
||||
model_arguments = ["--model-dir", models, "--embedding-worker", worker, "--cpu-threads", "1"]
|
||||
hits = run("semantic_search", ["search", "--db", db,
|
||||
"Wie beenden wir die Zusammenarbeit mit dem Lieferanten?", "--json", *model_arguments], True)
|
||||
if not hits or hits[0]["filename"] != "supplier.md" or "sixty days" not in hits[0]["snippet"] or not hits[0]["source"]:
|
||||
raise RuntimeError("Installed Google semantic search lost the supplier passage")
|
||||
code = run("code_search", ["search", "--db", db,
|
||||
"recover from transient network failures with increasing delays", "--code-only", "--json", *model_arguments], True)
|
||||
if not code or code[0]["filename"] != "retry.py" or code[0]["source"]["kind"] != "line":
|
||||
raise RuntimeError("Installed scoped code search lost its citation")
|
||||
restricted = run("strict_filter", ["search", "--db", db, "supplier cancellation",
|
||||
"--extension", "pdf", "--json", *model_arguments], True)
|
||||
if restricted:
|
||||
raise RuntimeError("Installed semantic search violated extension filter")
|
||||
evidence = {"platform": "linux-x86_64", "cpu_only": True,
|
||||
"archive_sha256": sha256(args.archive), "archive_bytes": args.archive.stat().st_size,
|
||||
"verified_files": len(manifest["files"]), "extraction_path_contains_spaces": True,
|
||||
"automatic_media_helper_discovery": "passed", "embedding_report": report,
|
||||
"semantic_top_filename": hits[0]["filename"], "code_source": code[0]["source"],
|
||||
"strict_filter": "passed", "command_elapsed_ms": timings,
|
||||
"timing_scope": "single deployment smoke samples; not retrieval/performance benchmarks",
|
||||
"limitations": ["GPU untested", "desktop untested", "clean-machine/ABI coverage untested", "broad retrieval quality untested"]}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"passed": True, "verified_files": evidence["verified_files"], "media_files": report["media_files"]}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Package an already-built Linux x86-64 headless development candidate.
|
||||
|
||||
No artifact is executed. readelf inspects dependency closure; native execution,
|
||||
ABI compatibility, GPU hardware and production installers remain release gates.
|
||||
"""
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_FILE = 256 * 1024 * 1024
|
||||
MAX_TOTAL = 1024 * 1024 * 1024
|
||||
CPU_VARIANTS = {"x64", "sse42", "sandybridge", "ivybridge", "piledriver", "haswell",
|
||||
"skylakex", "cannonlake", "cascadelake", "icelake", "cooperlake", "zen4",
|
||||
"alderlake", "sapphirerapids"}
|
||||
SYSTEM = {"libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0", "librt.so.1",
|
||||
"libmvec.so.1", "libstdc++.so.6", "libgcc_s.so.1", "ld-linux-x86-64.so.2"}
|
||||
|
||||
|
||||
def safe_name(name):
|
||||
if (not isinstance(name, str) or not name or "\\" in name or ":" in name
|
||||
or str(PurePosixPath(name)) != name or PurePosixPath(name).is_absolute()
|
||||
or any(part in {".", ".."} for part in name.split("/"))):
|
||||
raise ValueError("Unsafe package path")
|
||||
return name
|
||||
|
||||
|
||||
def regular(path, parent):
|
||||
path, parent = Path(path).absolute(), Path(parent).resolve()
|
||||
if not path.is_relative_to(parent) or not path.resolve().is_relative_to(parent):
|
||||
raise ValueError("Package source escapes its directory")
|
||||
candidate = path
|
||||
while candidate != parent:
|
||||
if candidate.is_symlink():
|
||||
raise ValueError("Package source contains a symlink")
|
||||
candidate = candidate.parent
|
||||
if not path.is_file() or not 0 < path.stat().st_size <= MAX_FILE:
|
||||
raise ValueError(f"Missing or oversized package source: {path.name}")
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def elf_x64(path):
|
||||
with path.open("rb") as stream:
|
||||
header = stream.read(64)
|
||||
if (len(header) != 64 or header[:7] != b"\x7fELF\x02\x01\x01"
|
||||
or struct.unpack_from("<H", header, 18)[0] != 62
|
||||
or struct.unpack_from("<H", header, 16)[0] not in {2, 3}):
|
||||
raise ValueError(f"Expected Linux x86-64 ELF artifact: {path.name}")
|
||||
|
||||
|
||||
def dynamic(path):
|
||||
output = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True, timeout=30)
|
||||
values = {}
|
||||
for key, value in re.findall(r"\((NEEDED|SONAME|RUNPATH|RPATH)\).*?\[([^\]]+)\]", output):
|
||||
values.setdefault(key, []).append(value)
|
||||
return values
|
||||
|
||||
|
||||
def runtime_files(runtime):
|
||||
inventory = runtime / "runtime-libraries.json"
|
||||
regular(inventory, runtime)
|
||||
if inventory.stat().st_size > 65536:
|
||||
raise ValueError("Oversized runtime inventory")
|
||||
inventory_bytes = inventory.read_bytes()
|
||||
manifest = json.loads(inventory_bytes)
|
||||
records = manifest.get("libraries") if isinstance(manifest, dict) else None
|
||||
if not isinstance(manifest, dict) or manifest.get("platform") != "linux" or not isinstance(records, list) or not 1 <= len(records) <= 64:
|
||||
raise ValueError("Invalid Linux runtime inventory")
|
||||
expected = {"searchi-gemma-worker": {"sha256": manifest.get("worker_sha256")}}
|
||||
core = {}
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
raise ValueError("Invalid library record")
|
||||
name = record.get("path")
|
||||
safe_name(name)
|
||||
match = re.fullmatch(r"lib(ggml|ggml-base|llama|mtmd)\.so(?:\.\d+)*", name)
|
||||
if match:
|
||||
if match[1] in core:
|
||||
raise ValueError("Duplicate runtime core library")
|
||||
core[match[1]] = name
|
||||
elif name != "libggml-vulkan.so" and name not in {f"libggml-cpu-{v}.so" for v in CPU_VARIANTS}:
|
||||
raise ValueError("Unapproved runtime library")
|
||||
if name in expected or type(record.get("bytes")) is not int or record["bytes"] <= 0:
|
||||
raise ValueError("Duplicate or invalid runtime library")
|
||||
expected[name] = record
|
||||
if set(core) != {"ggml", "ggml-base", "llama", "mtmd"} or not {"libggml-cpu-x64.so", "libggml-vulkan.so"}.issubset(expected):
|
||||
raise ValueError("Runtime lacks baseline CPU, Vulkan or core library")
|
||||
for name, record in expected.items():
|
||||
path = runtime / name
|
||||
regular(path, runtime)
|
||||
checksum = record.get("sha256")
|
||||
if not isinstance(checksum, str) or not re.fullmatch(r"[0-9a-f]{64}", checksum):
|
||||
raise ValueError("Invalid runtime checksum")
|
||||
if digest(path) != checksum or ("bytes" in record and path.stat().st_size != record["bytes"]):
|
||||
raise ValueError(f"Runtime integrity mismatch: {name}")
|
||||
record["bytes"] = path.stat().st_size
|
||||
elf_x64(path)
|
||||
metadata = dynamic(path)
|
||||
if metadata.get("RPATH") or metadata.get("RUNPATH") != ["$ORIGIN"]:
|
||||
raise ValueError("Runtime requires origin-only RUNPATH")
|
||||
allowed = set(expected) | SYSTEM
|
||||
if name == "libggml-vulkan.so":
|
||||
allowed.add("libvulkan.so.1")
|
||||
if set(metadata.get("NEEDED", [])) - allowed:
|
||||
raise ValueError(f"Unpackaged runtime dependency: {name}")
|
||||
if name in core.values() and metadata.get("SONAME") != [name]:
|
||||
raise ValueError("Core SONAME differs from packaged filename")
|
||||
expected[inventory.name] = {"sha256": hashlib.sha256(inventory_bytes).hexdigest(), "bytes": len(inventory_bytes)}
|
||||
return expected
|
||||
|
||||
|
||||
def media_files(root, media):
|
||||
manifest_path = root / "tools/media-runtime/manifest.json"
|
||||
regular(manifest_path, root)
|
||||
if manifest_path.stat().st_size > 65536:
|
||||
raise ValueError("Oversized media manifest")
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
artifact = manifest["platforms"]["linux-x86_64"]
|
||||
archive_name = safe_name(artifact["archive"])
|
||||
if "/" in archive_name or not archive_name.endswith(".tar.xz"):
|
||||
raise ValueError("Invalid media archive name")
|
||||
archive = media / archive_name
|
||||
regular(archive, media)
|
||||
if archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]:
|
||||
raise ValueError("Media archive provenance mismatch")
|
||||
selected = {}
|
||||
with tarfile.open(archive, "r:xz") as package:
|
||||
members = 0
|
||||
declared_total = 0
|
||||
for member in package:
|
||||
members += 1
|
||||
declared_total += member.size
|
||||
if (members > 10000 or member.size < 0 or member.size > MAX_FILE
|
||||
or declared_total > MAX_TOTAL):
|
||||
raise ValueError("Media archive exceeds member or size ceiling")
|
||||
basename = member.name.replace("\\", "/").split("/")[-1]
|
||||
if basename in {"ffmpeg", "ffprobe"}:
|
||||
target = f"bin/{basename}"
|
||||
elif basename.lower().startswith(("license", "copying")) or basename.lower() in {"readme.txt", "readme.md"}:
|
||||
target = "licenses/" + safe_name(member.name)
|
||||
else:
|
||||
continue
|
||||
if not member.isfile() or not 0 < member.size <= MAX_FILE or target in selected:
|
||||
raise ValueError("Invalid or duplicate media member")
|
||||
path = media / target
|
||||
regular(path, media)
|
||||
with package.extractfile(member) as stream:
|
||||
checksum = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
if path.stat().st_size != member.size or digest(path) != checksum:
|
||||
raise ValueError(f"Media helper/notice differs from pinned archive: {target}")
|
||||
selected[target] = {"sha256": checksum, "bytes": member.size}
|
||||
if not {"bin/ffmpeg", "bin/ffprobe"}.issubset(selected) or not any(n.startswith("licenses/") for n in selected):
|
||||
raise ValueError("Media archive lacks helpers or notices")
|
||||
selected = {"media-runtime/" + name: (media / name, record) for name, record in selected.items()}
|
||||
selected["media-runtime/manifest.json"] = (manifest_path, {"sha256": digest(manifest_path), "bytes": manifest_path.stat().st_size})
|
||||
return selected
|
||||
|
||||
|
||||
def package(root, executable, runtime, media, output, llama_license=None):
|
||||
root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output))
|
||||
regular(executable, executable.parent)
|
||||
elf_x64(executable)
|
||||
expected = runtime_files(runtime)
|
||||
cli_metadata = dynamic(executable)
|
||||
if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH")
|
||||
or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
|
||||
raise ValueError("CLI has unexpected native dependency or search path")
|
||||
selected = {name: (runtime / name, record) for name, record in expected.items()}
|
||||
selected.update(media_files(root, media))
|
||||
selected["searchi"] = (executable, {"sha256": digest(executable), "bytes": executable.stat().st_size})
|
||||
license_path = Path(llama_license) if llama_license else root / ".searchi/references/llama.cpp/LICENSE"
|
||||
for path, name, parent in [(root / "LICENSE", "LICENSE.txt", root), (license_path, "LLAMA-LICENSE.txt", license_path.parent)]:
|
||||
regular(path, parent)
|
||||
selected[name] = (path, {"sha256": digest(path), "bytes": path.stat().st_size})
|
||||
for name in ("media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"):
|
||||
elf_x64(selected[name][0])
|
||||
metadata = dynamic(selected[name][0])
|
||||
if metadata.get("RPATH") or metadata.get("RUNPATH") or set(metadata.get("NEEDED", [])) - SYSTEM:
|
||||
raise ValueError("Media helper has unpackaged dependencies")
|
||||
if sum(record["bytes"] for _, record in selected.values()) > MAX_TOTAL:
|
||||
raise ValueError("Package exceeds total size ceiling")
|
||||
if output.resolve() in {path.resolve() for path, _ in selected.values()}:
|
||||
raise ValueError("Package output would overwrite an input")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=output.parent, suffix=".partial", delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
try:
|
||||
records = []
|
||||
with temporary_path.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive:
|
||||
for name, (path, expected_record) in sorted(selected.items()):
|
||||
safe_name(name)
|
||||
regular(path, path.parent)
|
||||
# Bounded immutable snapshot: archive and manifest describe the same bytes.
|
||||
with path.open("rb") as stream:
|
||||
data = stream.read(MAX_FILE + 1)
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]:
|
||||
raise ValueError(f"Source changed during packaging: {name}")
|
||||
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
|
||||
add(archive, name, data, mode)
|
||||
records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)})
|
||||
instructions = ("searchi — Find what matters. Locally.\n\n"
|
||||
"Extract the entire archive into its own folder. This is a headless development candidate, not a production installer.\n"
|
||||
"Use ./searchi --help. Setup downloads about 865 MB of verified Google model assets:\n"
|
||||
"./searchi setup --model-dir ./models\n"
|
||||
"./searchi index --db ./searchi.sqlite --root /path/to/files\n"
|
||||
"./searchi embed --db ./searchi.sqlite --model-dir ./models --embedding-worker ./searchi-gemma-worker\n"
|
||||
"./searchi search --db ./searchi.sqlite 'what you remember' --model-dir ./models --embedding-worker ./searchi-gemma-worker\n"
|
||||
"Media helpers are discovered automatically beside this CLI. Ordinary search stays local.\n"
|
||||
"The normal runtime includes Vulkan and CPU fallback. Graphics drivers and a compatible system Vulkan loader are optional.\n"
|
||||
"Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n").encode()
|
||||
add(archive, "START-HERE.txt", instructions, 0o644)
|
||||
records.append({"path": "START-HERE.txt", "bytes": len(instructions),
|
||||
"sha256": hashlib.sha256(instructions).hexdigest(), "mode": oct(0o644)})
|
||||
add(archive, "PACKAGE.json", json.dumps({"platform": "linux", "architecture": "x86_64",
|
||||
"status": "unsigned headless development candidate; distribution ABI and GPU hardware validation pending",
|
||||
"system_libraries": sorted(SYSTEM), "optional_gpu_loader": "libvulkan.so.1",
|
||||
"files": records}, indent=2).encode(), 0o644)
|
||||
os.replace(temporary_path, output)
|
||||
finally:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
return "x86_64"
|
||||
|
||||
|
||||
def add(archive, name, data, mode):
|
||||
info = tarfile.TarInfo(safe_name(name))
|
||||
info.size, info.mode, info.mtime = len(data), mode, 0
|
||||
info.uid = info.gid = 0
|
||||
info.uname = info.gname = ""
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--executable", type=Path, default=ROOT / "target/release/searchi")
|
||||
parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto")
|
||||
parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime")
|
||||
parser.add_argument("--llama-license", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=ROOT / ".searchi/packages/searchi-linux-x86_64.tar.gz")
|
||||
args = parser.parse_args()
|
||||
package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license)
|
||||
print(f"Packaged Linux x86_64 development candidate: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,232 @@
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import package_linux as pack
|
||||
|
||||
|
||||
def elf(path, machine=62):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = bytearray(64)
|
||||
data[:7] = b"\x7fELF\x02\x01\x01"
|
||||
struct.pack_into("<HH", data, 16, 3, machine)
|
||||
path.write_bytes(data)
|
||||
|
||||
|
||||
class LinuxPackageTests(unittest.TestCase):
|
||||
def fixture(self, directory):
|
||||
root = Path(directory)
|
||||
runtime, media = root / "runtime", root / "media"
|
||||
executable = root / "build/searchi"
|
||||
elf(executable)
|
||||
(root / "LICENSE").write_text("application MIT notice")
|
||||
license = root / ".searchi/references/llama.cpp/LICENSE"
|
||||
license.parent.mkdir(parents=True)
|
||||
license.write_text("llama MIT notice")
|
||||
names = ["libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0", "libggml-cpu-x64.so", "libggml-vulkan.so"]
|
||||
elf(runtime / "searchi-gemma-worker")
|
||||
records = []
|
||||
for name in names:
|
||||
elf(runtime / name)
|
||||
records.append({"path": name, "bytes": 64, "sha256": pack.digest(runtime / name)})
|
||||
(runtime / "runtime-libraries.json").write_text(json.dumps({"platform": "linux", "worker_sha256": pack.digest(runtime / "searchi-gemma-worker"), "libraries": records}))
|
||||
media.mkdir()
|
||||
elf(media / "bin/ffmpeg")
|
||||
elf(media / "bin/ffprobe")
|
||||
notice = media / "licenses/upstream/LICENSE.txt"
|
||||
notice.parent.mkdir(parents=True)
|
||||
notice.write_text("LGPL notice")
|
||||
archive = media / "verified.tar.xz"
|
||||
with tarfile.open(archive, "w:xz") as output:
|
||||
for name, path in [("upstream/bin/ffmpeg", media / "bin/ffmpeg"), ("upstream/bin/ffprobe", media / "bin/ffprobe"), ("upstream/LICENSE.txt", notice)]:
|
||||
output.add(path, arcname=name)
|
||||
manifest = root / "tools/media-runtime/manifest.json"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_text(json.dumps({"provider": "test-fixture", "release": "fixture", "platforms": {"linux-x86_64": {"archive": archive.name, "bytes": archive.stat().st_size, "sha256": pack.digest(archive)}}}))
|
||||
return root, executable, runtime, media, root / "candidate.tar.gz"
|
||||
|
||||
@staticmethod
|
||||
def metadata(path):
|
||||
if path.name == "searchi-gemma-worker" or path.name.startswith("lib"):
|
||||
metadata = {"RUNPATH": ["$ORIGIN"], "NEEDED": ["libc.so.6"]}
|
||||
if path.name in {"libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0"}:
|
||||
metadata["SONAME"] = [path.name]
|
||||
if path.name == "libggml-vulkan.so":
|
||||
metadata["NEEDED"].append("libvulkan.so.1")
|
||||
return metadata
|
||||
return {"NEEDED": ["libc.so.6"]}
|
||||
|
||||
def run_package(self, fixture):
|
||||
with mock.patch.object(pack, "dynamic", side_effect=self.metadata):
|
||||
return pack.package(*fixture)
|
||||
|
||||
def test_archive_integrity_modes_no_userdata_and_reproducibility(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
root, executable, runtime, media, output = fixture
|
||||
(runtime / "settings.json").write_text("private")
|
||||
(media / "index.sqlite").write_text("private")
|
||||
self.run_package(fixture)
|
||||
first = output.read_bytes()
|
||||
self.run_package(fixture)
|
||||
self.assertEqual(first, output.read_bytes())
|
||||
with tarfile.open(output) as archive:
|
||||
self.assertNotIn("settings.json", archive.getnames())
|
||||
manifest = json.load(archive.extractfile("PACKAGE.json"))
|
||||
for record in manifest["files"]:
|
||||
member = archive.getmember(record["path"])
|
||||
data = archive.extractfile(member).read()
|
||||
self.assertEqual(record["sha256"], hashlib.sha256(data).hexdigest())
|
||||
self.assertEqual(member.uid, 0)
|
||||
self.assertEqual(member.mtime, 0)
|
||||
self.assertEqual(archive.getmember("searchi").mode, 0o755)
|
||||
self.assertEqual(archive.getmember("media-runtime/bin/ffmpeg").mode, 0o755)
|
||||
self.assertEqual(archive.getmember("LICENSE.txt").mode, 0o644)
|
||||
|
||||
def test_tampered_runtime_preserves_previous_archive(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
self.run_package(fixture)
|
||||
previous = fixture[-1].read_bytes()
|
||||
(fixture[2] / "libggml-cpu-x64.so").write_bytes(b"tampered")
|
||||
with self.assertRaisesRegex(ValueError, "integrity mismatch"):
|
||||
self.run_package(fixture)
|
||||
self.assertEqual(previous, fixture[-1].read_bytes())
|
||||
|
||||
def test_media_helper_tamper_is_rejected_against_pinned_archive(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
(fixture[3] / "bin/ffmpeg").write_bytes(b"tampered")
|
||||
with self.assertRaisesRegex(ValueError, "differs from pinned archive"):
|
||||
self.run_package(fixture)
|
||||
|
||||
def test_media_archive_provenance_tamper_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
with (fixture[3] / "verified.tar.xz").open("ab") as stream:
|
||||
stream.write(b"tampered")
|
||||
with self.assertRaisesRegex(ValueError, "provenance mismatch"):
|
||||
self.run_package(fixture)
|
||||
|
||||
def test_mixed_architecture_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
elf(fixture[1], machine=183)
|
||||
with self.assertRaisesRegex(ValueError, "x86-64 ELF"):
|
||||
self.run_package(fixture)
|
||||
|
||||
def test_inventory_traversal_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
path = fixture[2] / "runtime-libraries.json"
|
||||
manifest = json.loads(path.read_text())
|
||||
manifest["libraries"][0]["path"] = "../private.so"
|
||||
path.write_text(json.dumps(manifest))
|
||||
with self.assertRaisesRegex(ValueError, "Unsafe package path"):
|
||||
self.run_package(fixture)
|
||||
|
||||
def test_missing_baseline_cpu_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
path = fixture[2] / "runtime-libraries.json"
|
||||
manifest = json.loads(path.read_text())
|
||||
manifest["libraries"] = [r for r in manifest["libraries"] if r["path"] != "libggml-cpu-x64.so"]
|
||||
path.write_text(json.dumps(manifest))
|
||||
with self.assertRaisesRegex(ValueError, "lacks baseline"):
|
||||
self.run_package(fixture)
|
||||
|
||||
def test_worker_cannot_link_vulkan_loader(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
def metadata(path):
|
||||
result = self.metadata(path)
|
||||
if path.name == "searchi-gemma-worker":
|
||||
result["NEEDED"].append("libvulkan.so.1")
|
||||
return result
|
||||
with mock.patch.object(pack, "dynamic", side_effect=metadata), self.assertRaisesRegex(ValueError, "Unpackaged runtime"):
|
||||
pack.package(*fixture)
|
||||
|
||||
def test_external_runtime_search_path_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
with mock.patch.object(pack, "dynamic", return_value={"RUNPATH": ["/build/lib"]}), self.assertRaisesRegex(ValueError, "origin-only"):
|
||||
pack.package(*fixture)
|
||||
|
||||
def test_glibc_vector_math_is_declared_but_unknown_cli_dependency_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
def vector_math(path):
|
||||
result = self.metadata(path)
|
||||
if path.name == "searchi":
|
||||
result["NEEDED"].append("libmvec.so.1")
|
||||
return result
|
||||
with mock.patch.object(pack, "dynamic", side_effect=vector_math):
|
||||
pack.package(*fixture)
|
||||
with tarfile.open(fixture[-1]) as archive:
|
||||
manifest = json.load(archive.extractfile("PACKAGE.json"))
|
||||
self.assertIn("libmvec.so.1", manifest["system_libraries"])
|
||||
previous = fixture[-1].read_bytes()
|
||||
def unknown(path):
|
||||
result = vector_math(path)
|
||||
if path.name == "searchi":
|
||||
result["NEEDED"].append("libunpackaged.so.1")
|
||||
return result
|
||||
with mock.patch.object(pack, "dynamic", side_effect=unknown), self.assertRaisesRegex(ValueError, "unexpected native dependency"):
|
||||
pack.package(*fixture)
|
||||
self.assertEqual(previous, fixture[-1].read_bytes())
|
||||
|
||||
def test_readelf_parser_is_nonexecuting_and_preserves_dependencies(self):
|
||||
with mock.patch.object(pack.subprocess, "check_output", return_value="(NEEDED) Shared library: [libc.so.6]\n(RUNPATH) Library runpath: [$ORIGIN]\n") as process:
|
||||
self.assertEqual(pack.dynamic(Path("candidate")), {"NEEDED": ["libc.so.6"], "RUNPATH": ["$ORIGIN"]})
|
||||
self.assertEqual(process.call_args.args[0], ["readelf", "--dynamic", "candidate"])
|
||||
|
||||
def test_missing_pinned_media_notice_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
(fixture[3] / "licenses/upstream/LICENSE.txt").unlink()
|
||||
with self.assertRaisesRegex(ValueError, "Missing or oversized"):
|
||||
self.run_package(fixture)
|
||||
|
||||
def test_output_cannot_overwrite_cli(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root, executable, runtime, media, _ = self.fixture(directory)
|
||||
original = executable.read_bytes()
|
||||
with self.assertRaisesRegex(ValueError, "overwrite an input"):
|
||||
self.run_package((root, executable, runtime, media, executable))
|
||||
self.assertEqual(original, executable.read_bytes())
|
||||
|
||||
def test_changed_source_during_packaging_preserves_previous_archive(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
self.run_package(fixture)
|
||||
previous = fixture[-1].read_bytes()
|
||||
original_add = pack.add
|
||||
def mutate(archive, name, data, mode):
|
||||
original_add(archive, name, data, mode)
|
||||
if name == "LICENSE.txt":
|
||||
fixture[1].write_bytes(b"changed after preflight")
|
||||
with mock.patch.object(pack, "add", side_effect=mutate), self.assertRaisesRegex(ValueError, "Source changed"):
|
||||
self.run_package(fixture)
|
||||
self.assertEqual(previous, fixture[-1].read_bytes())
|
||||
|
||||
def test_unapproved_plugin_rejected_even_with_valid_hash(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fixture = self.fixture(directory)
|
||||
runtime = fixture[2]
|
||||
elf(runtime / "libarbitrary.so")
|
||||
inventory = runtime / "runtime-libraries.json"
|
||||
manifest = json.loads(inventory.read_text())
|
||||
manifest["libraries"].append({"path": "libarbitrary.so", "bytes": 64, "sha256": pack.digest(runtime / "libarbitrary.so")})
|
||||
inventory.write_text(json.dumps(manifest))
|
||||
with self.assertRaisesRegex(ValueError, "Unapproved runtime library"):
|
||||
self.run_package(fixture)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user