From 07bbcb76e809d18c0bf83a404da13e26108701a4 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 13:04:58 +0200 Subject: [PATCH] Reject stale native runtime build provenance before release staging --- docs/BUILDING.md | 5 ++ docs/ROADMAP.md | 3 + docs/RUNTIME-PROVENANCE.md | 55 +++++++++++++++++++ tools/embeddinggemma2/build_runtime.py | 12 +++- tools/embeddinggemma2/runtime_provenance.py | 34 ++++++++++++ tools/embeddinggemma2/stage_runtime.py | 6 ++ .../test_runtime_provenance.py | 39 +++++++++++++ tools/embeddinggemma2/test_stage_runtime.py | 9 ++- tools/package_linux.py | 6 +- tools/package_windows.py | 2 + tools/test_package_linux.py | 23 +++++++- tools/test_package_windows.py | 19 ++++++- 12 files changed, 204 insertions(+), 9 deletions(-) create mode 100644 docs/RUNTIME-PROVENANCE.md create mode 100644 tools/embeddinggemma2/runtime_provenance.py create mode 100644 tools/embeddinggemma2/test_runtime_provenance.py diff --git a/docs/BUILDING.md b/docs/BUILDING.md index ad2f6de..fd324e3 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -162,6 +162,11 @@ full extracted archive smoke. A legacy worker lacking `--role query` can build document vectors while failing foreground search. The archive smoke retains captured CLI error output when a command exits unsuccessfully. +Normal runtime inventories now include build-input provenance. Rebuild the +normal runtime before staging historical caches; source changes invalidate +their inventory even if the stored file checksums are valid. See +[runtime provenance](RUNTIME-PROVENANCE.md) for scope and actual validation. + The optional Linux missing-loader check requires explicit root execution and util-linux mount namespaces/chroot. It never escalates privileges and verifies a private namespace before creating an isolated filesystem: diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e58f03e..1085e96 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -119,6 +119,9 @@ release. Keep their evidence and unchecked production gates intact. modifying staged resources; corrupted/missing assets preserve existing worker. - [x] Transactional resource staging, verified copied bytes, process locking, interrupted-copy/swap recovery and backup preservation on failed rollback. +- [x] Normal Windows/Linux source-build provenance gate before staging and + archive preparation; stale inventories preserve existing resources/packages. + Actual normal builds verified. See [runtime provenance](RUNTIME-PROVENANCE.md). - [ ] Verify power-loss durability and release-tool interaction with resource directory swaps on each OS; fault-injected process interruption is narrower. - [x] Shared monotonic Basic/Debug search metrics, granular bounded engine traces, diff --git a/docs/RUNTIME-PROVENANCE.md b/docs/RUNTIME-PROVENANCE.md new file mode 100644 index 0000000..e883724 --- /dev/null +++ b/docs/RUNTIME-PROVENANCE.md @@ -0,0 +1,55 @@ +# Native runtime build-input provenance + +Checksum-valid historical runtime caches can be incompatible with the current +worker protocol. The Linux archive check detected this when an old worker +encoded documents but rejected the foreground `--role query` option. Source +release preparation now records and checks build-input identity before staging +or packaging a normal Windows/Linux runtime. + +`runtime-libraries.json` includes `build_provenance`, schema 1, the pinned +llama.cpp commit and SHA-256 of six bounded inputs: worker.cpp, native +CMakeLists.txt, build_runtime.py and the three runtime patches. Only CRLF line +endings in these build inputs are canonicalized to LF, allowing the same +source revision to compare across checkouts. Indexed user-file bytes and their +extraction provenance are unaffected. + +The builder captures provenance before CMake compilation and rejects changes +observed afterward before publishing a new inventory. Staging requires it for +normal distributions; Windows and Linux archive packagers require it as well. +Missing or stale provenance fails with an explicit normal-runtime rebuild +instruction before staged resources or an existing archive are replaced. +Internal explicit CPU tooling retains its legacy optional-inventory path. + +This is a stale-cache guard, not a signature, complete upstream source audit, +proof that a binary implements its claimed protocol, or a hardware-parity test. +Real query-role and installed-layout inference checks remain necessary. +Direct Tauri commands do not replace the documented build/stage preparation. +Existing published packages and user installations do not run this source-build +check and were not altered retroactively. + +## Recorded evidence, 2026-10-13 + +- 69 tests passed across provenance, builder, staging, Windows packaging and + Linux packaging suites; one platform-specific staging test remained skipped. + The same suites passed under optimized Python. Regressions reject a valid old + inventory without provenance, changed source and oversized inputs, while + preserving existing archives. Build-source CRLF/LF equivalence is checked. +- Actual normal Windows `auto` build and transactional staging passed. + Worker SHA-256 remains + `c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73`; + all 25 Windows DLL hashes also match the previously tested release exactly. + New staged inventory SHA-256: + `7fde510984b4635ce127d25ad91aab9a391300d7968ac71e4f201cc8edd052e8`. +- An actual historical extracted Windows inventory was rejected by normal + staging preflight for missing provenance. Its files were only read. +- Actual normal Linux `auto` build passed on Ubuntu 24.04 WSL. The packager + verified 21 runtime inventory/files and all six source identities against the + current repository, including checkout line-ending differences. + Worker SHA-256 remains + `b52d50a1e676bd4bf64124daff1c4ce83d86a8bde01e4eaf09838cbb24a4c28f`. + New inventory SHA-256: + `de6e06c68815b35d83c6acbde468d100413f47c74536f0d4219a9edcbeb01a86`. + +These normal builds retain GPU modules plus CPU fallback; no GPU inference was +executed. macOS inventory integration, cross-compilation and actual GPU checks +remain open. Published Windows and Linux assets retain their original hashes. diff --git a/tools/embeddinggemma2/build_runtime.py b/tools/embeddinggemma2/build_runtime.py index 4fc8096..f878eee 100644 --- a/tools/embeddinggemma2/build_runtime.py +++ b/tools/embeddinggemma2/build_runtime.py @@ -9,10 +9,11 @@ from pathlib import Path import shutil import subprocess import sys +from runtime_provenance import fingerprint, UPSTREAM_COMMIT ROOT = Path(__file__).resolve().parents[2] TAG = "b11461" -COMMIT = "4d756bc72bf00a4aacf410ae15a2d315f3db400d" +COMMIT = UPSTREAM_COMMIT WINDOWS_CRT_FILES = ( "concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", @@ -40,7 +41,7 @@ def elf_dynamic(path): return values -def bundle_linux_libraries(build): +def bundle_linux_libraries(build, provenance=None): """Flatten approved SONAME files beside the worker and record dependency closure. System libc/C++ ABI remains a declared distro compatibility gate. Vulkan's @@ -94,6 +95,7 @@ def bundle_linux_libraries(build): with path.open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest() inventory = {"platform": "linux", "worker_sha256": checksum(worker), + "build_provenance": provenance if provenance is not None else fingerprint(ROOT), "libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)} for _, path, _ in sorted(records)]} temporary = build / "runtime-libraries.json.partial" @@ -188,10 +190,13 @@ def main(): except ImportError as error: raise RuntimeError("Install CMake first (python -m pip install cmake)") from error worker_source = ROOT / "crates/searchi-semantic/native" + provenance = fingerprint(ROOT) run([cmake_exe, "-S", worker_source, "-B", build, "-DCMAKE_BUILD_TYPE=Release", f"-DLLAMA_CPP_DIR={source}", f"-DSEARCHI_ACCELERATION={args.backend}"]) run([cmake_exe, "--build", build, "--config", "Release", "--parallel", args.jobs]) + if fingerprint(ROOT) != provenance: + raise RuntimeError("Native build inputs changed during compilation; rerun the build") if vulkan_loader: for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]: shutil.copy2(vulkan_loader / name, build / "Release" / name) @@ -202,13 +207,14 @@ def main(): with path.open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest() inventory = {"worker_sha256": checksum(directory / "searchi-gemma-worker.exe"), + "build_provenance": provenance, "libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)} for path in sorted(directory.glob("*.dll"))]} temporary = directory / "runtime-libraries.json.partial" temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8") os.replace(temporary, directory / "runtime-libraries.json") if linux_dispatch(args.backend): - bundle_linux_libraries(build) + bundle_linux_libraries(build, provenance) print(f"Runtime build complete: {build}") diff --git a/tools/embeddinggemma2/runtime_provenance.py b/tools/embeddinggemma2/runtime_provenance.py new file mode 100644 index 0000000..6c8b356 --- /dev/null +++ b/tools/embeddinggemma2/runtime_provenance.py @@ -0,0 +1,34 @@ +"""Build-input identity for detecting stale, otherwise checksum-valid runtimes.""" +import hashlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +UPSTREAM_COMMIT = "4d756bc72bf00a4aacf410ae15a2d315f3db400d" +INPUTS = ( + "crates/searchi-semantic/native/worker.cpp", + "crates/searchi-semantic/native/CMakeLists.txt", + "tools/embeddinggemma2/build_runtime.py", + "tools/embeddinggemma2/no-logits.patch", + "tools/embeddinggemma2/cpu-dispatch.patch", + "tools/embeddinggemma2/cpu-only-loader.patch", +) + + +def fingerprint(root=ROOT): + hashes = {} + for name in INPUTS: + path = Path(root) / name + with path.open("rb") as source: + data = source.read(1024 * 1024 + 1) + if not data or len(data) > 1024 * 1024: + raise ValueError("Native build input is empty or oversized") + # Only build-source line endings are canonicalized; indexed user bytes + # and extraction provenance are never processed by this helper. + hashes[name] = hashlib.sha256(data.replace(b"\r\n", b"\n")).hexdigest() + return {"schema": 1, "upstream_commit": UPSTREAM_COMMIT, + "normalization": "CRLF-to-LF build inputs only", "inputs": hashes} + + +def verify(manifest, root=ROOT): + if not isinstance(manifest.get("build_provenance"), dict) or manifest["build_provenance"] != fingerprint(root): + raise ValueError("Runtime build provenance missing or stale; rebuild the normal native runtime") diff --git a/tools/embeddinggemma2/stage_runtime.py b/tools/embeddinggemma2/stage_runtime.py index 7a82229..5edf24b 100644 --- a/tools/embeddinggemma2/stage_runtime.py +++ b/tools/embeddinggemma2/stage_runtime.py @@ -10,6 +10,7 @@ import shutil import stat import time import uuid +from runtime_provenance import verify as verify_provenance ROOT = Path(__file__).resolve().parents[2] CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"} @@ -339,6 +340,11 @@ def verify_runtime(source, required=False): manifest = json.loads(inventory.read_text(encoding="utf-8")) if not isinstance(manifest, dict): raise RuntimeError("Invalid runtime dependency inventory") + if required or "build_provenance" in manifest: + try: + verify_provenance(manifest, ROOT) + except ValueError as error: + raise RuntimeError(str(error)) from error with source.open("rb") as stream: if hashlib.file_digest(stream, "sha256").hexdigest() != manifest.get("worker_sha256"): raise RuntimeError("Runtime worker checksum mismatch") diff --git a/tools/embeddinggemma2/test_runtime_provenance.py b/tools/embeddinggemma2/test_runtime_provenance.py new file mode 100644 index 0000000..f010743 --- /dev/null +++ b/tools/embeddinggemma2/test_runtime_provenance.py @@ -0,0 +1,39 @@ +from pathlib import Path +import tempfile +import unittest + +import runtime_provenance as provenance + + +class ProvenanceTests(unittest.TestCase): + def test_line_endings_equivalent_but_changed_source_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in provenance.INPUTS: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"first\nsecond\n") + manifest = {"build_provenance": provenance.fingerprint(root)} + for name in provenance.INPUTS: + (root / name).write_bytes(b"first\r\nsecond\r\n") + provenance.verify(manifest, root) + (root / provenance.INPUTS[0]).write_bytes(b"different source\n") + with self.assertRaisesRegex(ValueError, "stale"): + provenance.verify(manifest, root) + + def test_missing_old_inventory_rejected_and_inputs_bounded(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in provenance.INPUTS: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"source\n") + with self.assertRaisesRegex(ValueError, "missing or stale"): + provenance.verify({"worker_sha256": "old-valid-hash"}, root) + (root / provenance.INPUTS[0]).write_bytes(b"x" * (1024 * 1024 + 1)) + with self.assertRaisesRegex(ValueError, "oversized"): + provenance.fingerprint(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/embeddinggemma2/test_stage_runtime.py b/tools/embeddinggemma2/test_stage_runtime.py index 55825b4..ab603da 100644 --- a/tools/embeddinggemma2/test_stage_runtime.py +++ b/tools/embeddinggemma2/test_stage_runtime.py @@ -9,6 +9,7 @@ import unittest from unittest.mock import patch import stage_runtime +import runtime_provenance class StagingTests(unittest.TestCase): @@ -28,7 +29,7 @@ class StagingTests(unittest.TestCase): records.append({"path": name, "bytes": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}) inventory = runtime / "runtime-libraries.json" - inventory.write_text(json.dumps({"platform": "linux", "worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records})) + inventory.write_text(json.dumps({"platform": "linux", "worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "build_provenance": runtime_provenance.fingerprint(), "libraries": records})) dependencies = stage_runtime.verify_runtime(worker, True) destination = root / "resources" destination.mkdir() @@ -419,6 +420,10 @@ class StagingTests(unittest.TestCase): def test_missing_crt_notice_preserves_existing_staged_worker(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) + for name in runtime_provenance.INPUTS: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((runtime_provenance.ROOT / name).read_bytes()) runtime = root / ".searchi/runtime/auto/Release" runtime.mkdir(parents=True) worker = runtime / "searchi-gemma-worker.exe" @@ -429,7 +434,7 @@ class StagingTests(unittest.TestCase): data = name.encode() (runtime / name).write_bytes(data) records.append({"path": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}) - (runtime / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records})) + (runtime / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "build_provenance": runtime_provenance.fingerprint(root), "libraries": records})) license_path = root / ".searchi/references/llama.cpp/LICENSE" license_path.parent.mkdir(parents=True) license_path.write_text("license") diff --git a/tools/package_linux.py b/tools/package_linux.py index f2d7122..611e911 100644 --- a/tools/package_linux.py +++ b/tools/package_linux.py @@ -16,6 +16,7 @@ import struct import subprocess import tarfile import tempfile +from embeddinggemma2.runtime_provenance import verify as verify_provenance ROOT = Path(__file__).resolve().parents[1] MAX_FILE = 256 * 1024 * 1024 @@ -70,7 +71,7 @@ def dynamic(path): return values -def runtime_files(runtime): +def runtime_files(runtime, root=ROOT): inventory = runtime / "runtime-libraries.json" regular(inventory, runtime) if inventory.stat().st_size > 65536: @@ -80,6 +81,7 @@ def runtime_files(runtime): records = manifest.get("libraries") if isinstance(manifest, dict) else None if not isinstance(manifest, dict) or manifest.get("platform") != "linux" or not isinstance(records, list) or not 1 <= len(records) <= 64: raise ValueError("Invalid Linux runtime inventory") + verify_provenance(manifest, root) expected = {"searchi-gemma-worker": {"sha256": manifest.get("worker_sha256")}} core = {} for record in records: @@ -176,7 +178,7 @@ def package(root, executable, runtime, media, output, llama_license=None, compre root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output)) regular(executable, executable.parent) elf_x64(executable) - expected = runtime_files(runtime) + expected = runtime_files(runtime, root) cli_metadata = dynamic(executable) if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH") or set(cli_metadata.get("NEEDED", [])) - SYSTEM): diff --git a/tools/package_windows.py b/tools/package_windows.py index 1368cc0..b5fb9cc 100644 --- a/tools/package_windows.py +++ b/tools/package_windows.py @@ -13,6 +13,7 @@ import struct import stat import tempfile import zipfile +from embeddinggemma2.runtime_provenance import verify as verify_provenance ROOT = Path(__file__).resolve().parents[1] CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"} @@ -114,6 +115,7 @@ def package(root, executable, output, cli=None): manifest = json.loads(inventory_bytes) if not isinstance(manifest, dict): raise ValueError("Invalid runtime dependency inventory") + verify_provenance(manifest, root) libraries = manifest.get("libraries") if not isinstance(libraries, list) or not 1 <= len(libraries) <= 64: raise ValueError("Invalid runtime dependency inventory") diff --git a/tools/test_package_linux.py b/tools/test_package_linux.py index bb2835b..9dca311 100644 --- a/tools/test_package_linux.py +++ b/tools/test_package_linux.py @@ -9,6 +9,7 @@ import unittest from unittest import mock import package_linux as pack +from embeddinggemma2 import runtime_provenance as provenance def elf(path, machine=62): @@ -22,6 +23,10 @@ def elf(path, machine=62): class LinuxPackageTests(unittest.TestCase): def fixture(self, directory): root = Path(directory) + for name in provenance.INPUTS: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((provenance.ROOT / name).read_bytes()) runtime, media = root / "runtime", root / "media" executable = root / "build/searchi" elf(executable) @@ -35,7 +40,7 @@ class LinuxPackageTests(unittest.TestCase): for name in names: elf(runtime / name) records.append({"path": name, "bytes": 64, "sha256": pack.digest(runtime / name)}) - (runtime / "runtime-libraries.json").write_text(json.dumps({"platform": "linux", "worker_sha256": pack.digest(runtime / "searchi-gemma-worker"), "libraries": records})) + (runtime / "runtime-libraries.json").write_text(json.dumps({"platform": "linux", "worker_sha256": pack.digest(runtime / "searchi-gemma-worker"), "build_provenance": provenance.fingerprint(root), "libraries": records})) media.mkdir() elf(media / "bin/ffmpeg") elf(media / "bin/ffprobe") @@ -115,6 +120,22 @@ class LinuxPackageTests(unittest.TestCase): compressed = {m.name: (archive.extractfile(m).read(), m.mode) for m in archive} self.assertEqual(original, compressed) + def test_old_or_stale_build_provenance_preserves_previous_archive(self): + for mutation in ("missing", "source_changed"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + fixture = self.fixture(directory) + fixture[-1].write_bytes(b"previous archive") + if mutation == "missing": + inventory = fixture[2] / "runtime-libraries.json" + manifest = json.loads(inventory.read_text()) + del manifest["build_provenance"] + inventory.write_text(json.dumps(manifest)) + else: + (fixture[0] / provenance.INPUTS[0]).write_bytes(b"changed worker source") + with self.assertRaisesRegex(ValueError, "provenance"): + self.run_package(fixture) + self.assertEqual(fixture[-1].read_bytes(), b"previous archive") + def test_media_helper_tamper_is_rejected_against_pinned_archive(self): with tempfile.TemporaryDirectory() as directory: fixture = self.fixture(directory) diff --git a/tools/test_package_windows.py b/tools/test_package_windows.py index 3d43af5..59dc8e7 100644 --- a/tools/test_package_windows.py +++ b/tools/test_package_windows.py @@ -8,6 +8,7 @@ from unittest import mock import zipfile from package_windows import package, CRT_LIBRARIES +from embeddinggemma2 import runtime_provenance as provenance def executable(path, machine=0x8664): @@ -26,7 +27,7 @@ def inventory(resources): for name in ["ggml-cpu-x64.dll", "ggml-vulkan.dll", "vulkan-1.dll", "ggml.dll", "ggml-base.dll", "llama.dll", "mtmd.dll", *sorted(CRT_LIBRARIES)]: data = (resources / name).read_bytes() records.append({"path": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}) - (resources / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records})) + (resources / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "build_provenance": provenance.fingerprint(resources.parents[3]), "libraries": records})) class PackageTests(unittest.TestCase): @@ -48,8 +49,24 @@ class PackageTests(unittest.TestCase): package(root, app, output, cli) self.assertEqual(output.read_bytes(), previous) + def test_old_inventory_rejected_without_replacing_archive(self): + with tempfile.TemporaryDirectory() as directory: + root, resources, app, output = self.fixture(directory) + output.write_bytes(b"previous archive") + inventory = resources / "runtime-libraries.json" + manifest = json.loads(inventory.read_text()) + del manifest["build_provenance"] + inventory.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "provenance"): + package(root, app, output) + self.assertEqual(output.read_bytes(), b"previous archive") + def fixture(self, directory): root = Path(directory) + for name in provenance.INPUTS: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((provenance.ROOT / name).read_bytes()) resources = root / "apps/desktop/src-tauri/resources" app = root / "build/app.exe" executable(app)