karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Reject stale native runtime build provenance before release staging

Browse Source
This commit is contained in:
karl committed 2026-10-13 13:04:58 +02:00
1 parent ea2c3f4691
commit 07bbcb76e8
12 files changed
+204 -9
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/embeddinggemma2/build_runtime.py
+9 -3
View File
@@ -9,10 +9,11 @@ from pathlib import Path
import shutil
import subprocess
import sys
from runtime_provenance import fingerprint, UPSTREAM_COMMIT
ROOT = Path(__file__).resolve().parents[2]
TAG = "b11461"
COMMIT = "4d756bc72bf00a4aacf410ae15a2d315f3db400d"
COMMIT = UPSTREAM_COMMIT
WINDOWS_CRT_FILES = (
"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll",
"msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll",
@@ -40,7 +41,7 @@ def elf_dynamic(path):
return values
def bundle_linux_libraries(build):
def bundle_linux_libraries(build, provenance=None):
"""Flatten approved SONAME files beside the worker and record dependency closure.
System libc/C++ ABI remains a declared distro compatibility gate. Vulkan's
@@ -94,6 +95,7 @@ def bundle_linux_libraries(build):
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
inventory = {"platform": "linux", "worker_sha256": checksum(worker),
"build_provenance": provenance if provenance is not None else fingerprint(ROOT),
"libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)}
for _, path, _ in sorted(records)]}
temporary = build / "runtime-libraries.json.partial"
@@ -188,10 +190,13 @@ def main():
except ImportError as error:
raise RuntimeError("Install CMake first (python -m pip install cmake)") from error
worker_source = ROOT / "crates/searchi-semantic/native"
provenance = fingerprint(ROOT)
run([cmake_exe, "-S", worker_source, "-B", build,
"-DCMAKE_BUILD_TYPE=Release", f"-DLLAMA_CPP_DIR={source}",
f"-DSEARCHI_ACCELERATION={args.backend}"])
run([cmake_exe, "--build", build, "--config", "Release", "--parallel", args.jobs])
if fingerprint(ROOT) != provenance:
raise RuntimeError("Native build inputs changed during compilation; rerun the build")
if vulkan_loader:
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
shutil.copy2(vulkan_loader / name, build / "Release" / name)
@@ -202,13 +207,14 @@ def main():
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
inventory = {"worker_sha256": checksum(directory / "searchi-gemma-worker.exe"),
"build_provenance": provenance,
"libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)}
for path in sorted(directory.glob("*.dll"))]}
temporary = directory / "runtime-libraries.json.partial"
temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8")
os.replace(temporary, directory / "runtime-libraries.json")
if linux_dispatch(args.backend):
bundle_linux_libraries(build)
bundle_linux_libraries(build, provenance)
print(f"Runtime build complete: {build}")
tools/embeddinggemma2/runtime_provenance.py
+34
View File
@@ -0,0 +1,34 @@
"""Build-input identity for detecting stale, otherwise checksum-valid runtimes."""
import hashlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
UPSTREAM_COMMIT = "4d756bc72bf00a4aacf410ae15a2d315f3db400d"
INPUTS = (
"crates/searchi-semantic/native/worker.cpp",
"crates/searchi-semantic/native/CMakeLists.txt",
"tools/embeddinggemma2/build_runtime.py",
"tools/embeddinggemma2/no-logits.patch",
"tools/embeddinggemma2/cpu-dispatch.patch",
"tools/embeddinggemma2/cpu-only-loader.patch",
)
def fingerprint(root=ROOT):
hashes = {}
for name in INPUTS:
path = Path(root) / name
with path.open("rb") as source:
data = source.read(1024 * 1024 + 1)
if not data or len(data) > 1024 * 1024:
raise ValueError("Native build input is empty or oversized")
# Only build-source line endings are canonicalized; indexed user bytes
# and extraction provenance are never processed by this helper.
hashes[name] = hashlib.sha256(data.replace(b"\r\n", b"\n")).hexdigest()
return {"schema": 1, "upstream_commit": UPSTREAM_COMMIT,
"normalization": "CRLF-to-LF build inputs only", "inputs": hashes}
def verify(manifest, root=ROOT):
if not isinstance(manifest.get("build_provenance"), dict) or manifest["build_provenance"] != fingerprint(root):
raise ValueError("Runtime build provenance missing or stale; rebuild the normal native runtime")
tools/embeddinggemma2/stage_runtime.py
+6
View File
@@ -10,6 +10,7 @@ import shutil
import stat
import time
import uuid
from runtime_provenance import verify as verify_provenance
ROOT = Path(__file__).resolve().parents[2]
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
@@ -339,6 +340,11 @@ def verify_runtime(source, required=False):
manifest = json.loads(inventory.read_text(encoding="utf-8"))
if not isinstance(manifest, dict):
raise RuntimeError("Invalid runtime dependency inventory")
if required or "build_provenance" in manifest:
try:
verify_provenance(manifest, ROOT)
except ValueError as error:
raise RuntimeError(str(error)) from error
with source.open("rb") as stream:
if hashlib.file_digest(stream, "sha256").hexdigest() != manifest.get("worker_sha256"):
raise RuntimeError("Runtime worker checksum mismatch")
tools/embeddinggemma2/test_runtime_provenance.py
+39
View File
@@ -0,0 +1,39 @@
from pathlib import Path
import tempfile
import unittest
import runtime_provenance as provenance
class ProvenanceTests(unittest.TestCase):
def test_line_endings_equivalent_but_changed_source_rejected(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in provenance.INPUTS:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"first\nsecond\n")
manifest = {"build_provenance": provenance.fingerprint(root)}
for name in provenance.INPUTS:
(root / name).write_bytes(b"first\r\nsecond\r\n")
provenance.verify(manifest, root)
(root / provenance.INPUTS[0]).write_bytes(b"different source\n")
with self.assertRaisesRegex(ValueError, "stale"):
provenance.verify(manifest, root)
def test_missing_old_inventory_rejected_and_inputs_bounded(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in provenance.INPUTS:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"source\n")
with self.assertRaisesRegex(ValueError, "missing or stale"):
provenance.verify({"worker_sha256": "old-valid-hash"}, root)
(root / provenance.INPUTS[0]).write_bytes(b"x" * (1024 * 1024 + 1))
with self.assertRaisesRegex(ValueError, "oversized"):
provenance.fingerprint(root)
if __name__ == "__main__":
unittest.main()
tools/embeddinggemma2/test_stage_runtime.py
+7 -2
View File
@@ -9,6 +9,7 @@ import unittest
from unittest.mock import patch
import stage_runtime
import runtime_provenance
class StagingTests(unittest.TestCase):
@@ -28,7 +29,7 @@ class StagingTests(unittest.TestCase):
records.append({"path": name, "bytes": path.stat().st_size,
"sha256": hashlib.sha256(path.read_bytes()).hexdigest()})
inventory = runtime / "runtime-libraries.json"
inventory.write_text(json.dumps({"platform": "linux", "worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records}))
inventory.write_text(json.dumps({"platform": "linux", "worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "build_provenance": runtime_provenance.fingerprint(), "libraries": records}))
dependencies = stage_runtime.verify_runtime(worker, True)
destination = root / "resources"
destination.mkdir()
@@ -419,6 +420,10 @@ class StagingTests(unittest.TestCase):
def test_missing_crt_notice_preserves_existing_staged_worker(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in runtime_provenance.INPUTS:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((runtime_provenance.ROOT / name).read_bytes())
runtime = root / ".searchi/runtime/auto/Release"
runtime.mkdir(parents=True)
worker = runtime / "searchi-gemma-worker.exe"
@@ -429,7 +434,7 @@ class StagingTests(unittest.TestCase):
data = name.encode()
(runtime / name).write_bytes(data)
records.append({"path": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})
(runtime / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records}))
(runtime / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "build_provenance": runtime_provenance.fingerprint(root), "libraries": records}))
license_path = root / ".searchi/references/llama.cpp/LICENSE"
license_path.parent.mkdir(parents=True)
license_path.write_text("license")
tools/package_linux.py
+4 -2
View File
@@ -16,6 +16,7 @@ import struct
import subprocess
import tarfile
import tempfile
from embeddinggemma2.runtime_provenance import verify as verify_provenance
ROOT = Path(__file__).resolve().parents[1]
MAX_FILE = 256 * 1024 * 1024
@@ -70,7 +71,7 @@ def dynamic(path):
return values
def runtime_files(runtime):
def runtime_files(runtime, root=ROOT):
inventory = runtime / "runtime-libraries.json"
regular(inventory, runtime)
if inventory.stat().st_size > 65536:
@@ -80,6 +81,7 @@ def runtime_files(runtime):
records = manifest.get("libraries") if isinstance(manifest, dict) else None
if not isinstance(manifest, dict) or manifest.get("platform") != "linux" or not isinstance(records, list) or not 1 <= len(records) <= 64:
raise ValueError("Invalid Linux runtime inventory")
verify_provenance(manifest, root)
expected = {"searchi-gemma-worker": {"sha256": manifest.get("worker_sha256")}}
core = {}
for record in records:
@@ -176,7 +178,7 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output))
regular(executable, executable.parent)
elf_x64(executable)
expected = runtime_files(runtime)
expected = runtime_files(runtime, root)
cli_metadata = dynamic(executable)
if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH")
or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
tools/package_windows.py
+2
View File
@@ -13,6 +13,7 @@ import struct
import stat
import tempfile
import zipfile
from embeddinggemma2.runtime_provenance import verify as verify_provenance
ROOT = Path(__file__).resolve().parents[1]
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
@@ -114,6 +115,7 @@ def package(root, executable, output, cli=None):
manifest = json.loads(inventory_bytes)
if not isinstance(manifest, dict):
raise ValueError("Invalid runtime dependency inventory")
verify_provenance(manifest, root)
libraries = manifest.get("libraries")
if not isinstance(libraries, list) or not 1 <= len(libraries) <= 64:
raise ValueError("Invalid runtime dependency inventory")
tools/test_package_linux.py
+22 -1
View File
@@ -9,6 +9,7 @@ import unittest
from unittest import mock
import package_linux as pack
from embeddinggemma2 import runtime_provenance as provenance
def elf(path, machine=62):
@@ -22,6 +23,10 @@ def elf(path, machine=62):
class LinuxPackageTests(unittest.TestCase):
def fixture(self, directory):
root = Path(directory)
for name in provenance.INPUTS:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((provenance.ROOT / name).read_bytes())
runtime, media = root / "runtime", root / "media"
executable = root / "build/searchi"
elf(executable)
@@ -35,7 +40,7 @@ class LinuxPackageTests(unittest.TestCase):
for name in names:
elf(runtime / name)
records.append({"path": name, "bytes": 64, "sha256": pack.digest(runtime / name)})
(runtime / "runtime-libraries.json").write_text(json.dumps({"platform": "linux", "worker_sha256": pack.digest(runtime / "searchi-gemma-worker"), "libraries": records}))
(runtime / "runtime-libraries.json").write_text(json.dumps({"platform": "linux", "worker_sha256": pack.digest(runtime / "searchi-gemma-worker"), "build_provenance": provenance.fingerprint(root), "libraries": records}))
media.mkdir()
elf(media / "bin/ffmpeg")
elf(media / "bin/ffprobe")
@@ -115,6 +120,22 @@ class LinuxPackageTests(unittest.TestCase):
compressed = {m.name: (archive.extractfile(m).read(), m.mode) for m in archive}
self.assertEqual(original, compressed)
def test_old_or_stale_build_provenance_preserves_previous_archive(self):
for mutation in ("missing", "source_changed"):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
fixture = self.fixture(directory)
fixture[-1].write_bytes(b"previous archive")
if mutation == "missing":
inventory = fixture[2] / "runtime-libraries.json"
manifest = json.loads(inventory.read_text())
del manifest["build_provenance"]
inventory.write_text(json.dumps(manifest))
else:
(fixture[0] / provenance.INPUTS[0]).write_bytes(b"changed worker source")
with self.assertRaisesRegex(ValueError, "provenance"):
self.run_package(fixture)
self.assertEqual(fixture[-1].read_bytes(), b"previous archive")
def test_media_helper_tamper_is_rejected_against_pinned_archive(self):
with tempfile.TemporaryDirectory() as directory:
fixture = self.fixture(directory)
tools/test_package_windows.py
+18 -1
View File
@@ -8,6 +8,7 @@ from unittest import mock
import zipfile
from package_windows import package, CRT_LIBRARIES
from embeddinggemma2 import runtime_provenance as provenance
def executable(path, machine=0x8664):
@@ -26,7 +27,7 @@ def inventory(resources):
for name in ["ggml-cpu-x64.dll", "ggml-vulkan.dll", "vulkan-1.dll", "ggml.dll", "ggml-base.dll", "llama.dll", "mtmd.dll", *sorted(CRT_LIBRARIES)]:
data = (resources / name).read_bytes()
records.append({"path": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})
(resources / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "libraries": records}))
(resources / "runtime-libraries.json").write_text(json.dumps({"worker_sha256": hashlib.sha256(worker.read_bytes()).hexdigest(), "build_provenance": provenance.fingerprint(resources.parents[3]), "libraries": records}))
class PackageTests(unittest.TestCase):
@@ -48,8 +49,24 @@ class PackageTests(unittest.TestCase):
package(root, app, output, cli)
self.assertEqual(output.read_bytes(), previous)
def test_old_inventory_rejected_without_replacing_archive(self):
with tempfile.TemporaryDirectory() as directory:
root, resources, app, output = self.fixture(directory)
output.write_bytes(b"previous archive")
inventory = resources / "runtime-libraries.json"
manifest = json.loads(inventory.read_text())
del manifest["build_provenance"]
inventory.write_text(json.dumps(manifest))
with self.assertRaisesRegex(ValueError, "provenance"):
package(root, app, output)
self.assertEqual(output.read_bytes(), b"previous archive")
def fixture(self, directory):
root = Path(directory)
for name in provenance.INPUTS:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((provenance.ROOT / name).read_bytes())
resources = root / "apps/desktop/src-tauri/resources"
app = root / "build/app.exe"
executable(app)