From a8080f061f74f7bc056228df0cbe5e98c6fb0930 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 12:31:17 +0200 Subject: [PATCH] Support reproducible XZ Linux archives without changing runtime contents --- docs/BUILDING.md | 6 ++++++ tools/linux_archive_smoke.py | 2 +- tools/package_linux.py | 14 ++++++++++---- tools/test_package_linux.py | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 4b0d94f..506df82 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -133,6 +133,7 @@ an unsigned headless development candidate: ```sh python3 tools/package_linux.py +python3 tools/package_linux.py --compression xz python3 tools/embeddinggemma2/linux_smoke.py --worker .searchi/runtime/auto/searchi-gemma-worker --models .searchi/models/embeddinggemma2 --output .searchi/packages/linux-worker-smoke.json python3 tools/embeddinggemma2/linux_smoke.py --worker .searchi/runtime/auto/searchi-gemma-worker --models .searchi/models/embeddinggemma2 --output .searchi/packages/linux-baseline-smoke.json --baseline ``` @@ -148,6 +149,11 @@ They do not exercise GPU providers or certify older physical CPUs, installers or desktop behavior. Archive extraction and installed CLI search must also be verified before a release gate is closed. +The default archive uses gzip; `--compression xz` produces `.tar.xz` with the +same normalized TAR members, file checksums and executable modes. XZ uses the +standard library's preset 6. The extraction smoke accepts either format. A +smaller download does not change runtime behavior or ABI requirements. + The optional Linux missing-loader check requires explicit root execution and util-linux mount namespaces/chroot. It never escalates privileges and verifies a private namespace before creating an isolated filesystem: diff --git a/tools/linux_archive_smoke.py b/tools/linux_archive_smoke.py index 3830474..afda532 100644 --- a/tools/linux_archive_smoke.py +++ b/tools/linux_archive_smoke.py @@ -25,7 +25,7 @@ def sha256(path): def extract_checked(archive_path, destination): from package_linux import safe_name - with tarfile.open(archive_path, "r:gz") as archive: + with tarfile.open(archive_path, "r:*") as archive: members = archive.getmembers() if not 1 <= len(members) <= 128 or len({item.name for item in members}) != len(members): raise ValueError("Invalid candidate member inventory") diff --git a/tools/package_linux.py b/tools/package_linux.py index fe59c55..f2d7122 100644 --- a/tools/package_linux.py +++ b/tools/package_linux.py @@ -8,6 +8,7 @@ import gzip import hashlib import io import json +import lzma import os from pathlib import Path, PurePosixPath import re @@ -169,7 +170,9 @@ def media_files(root, media): return selected -def package(root, executable, runtime, media, output, llama_license=None): +def package(root, executable, runtime, media, output, llama_license=None, compression="gzip"): + if compression not in {"gzip", "xz"}: + raise ValueError("Unsupported package compression") root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output)) regular(executable, executable.parent) elf_x64(executable) @@ -199,7 +202,7 @@ def package(root, executable, runtime, media, output, llama_license=None): temporary_path = Path(temporary.name) try: records = [] - with temporary_path.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive: + with temporary_path.open("wb") as raw, (lzma.LZMAFile(raw, "w", preset=6) if compression == "xz" else gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0)) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive: for name, (path, expected_record) in sorted(selected.items()): safe_name(name) regular(path, path.parent) @@ -249,9 +252,12 @@ def main(): parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto") parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime") parser.add_argument("--llama-license", type=Path) - parser.add_argument("--output", type=Path, default=ROOT / ".searchi/packages/searchi-linux-x86_64.tar.gz") + parser.add_argument("--compression", choices=["gzip", "xz"], default="gzip") + parser.add_argument("--output", type=Path) args = parser.parse_args() - package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license) + if args.output is None: + args.output = ROOT / (".searchi/packages/searchi-linux-x86_64.tar.xz" if args.compression == "xz" else ".searchi/packages/searchi-linux-x86_64.tar.gz") + package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression) print(f"Packaged Linux x86_64 development candidate: {args.output}") diff --git a/tools/test_package_linux.py b/tools/test_package_linux.py index 4c30756..bb2835b 100644 --- a/tools/test_package_linux.py +++ b/tools/test_package_linux.py @@ -99,6 +99,22 @@ class LinuxPackageTests(unittest.TestCase): self.run_package(fixture) self.assertEqual(previous, fixture[-1].read_bytes()) + def test_xz_preserves_members_checksums_modes_and_reproducibility(self): + with tempfile.TemporaryDirectory() as directory: + fixture = self.fixture(directory) + self.run_package(fixture) + with tarfile.open(fixture[-1]) as archive: + original = {m.name: (archive.extractfile(m).read(), m.mode) for m in archive} + with mock.patch.object(pack, "dynamic", side_effect=self.metadata): + pack.package(*fixture, compression="xz") + first = fixture[-1].read_bytes() + pack.package(*fixture, compression="xz") + self.assertEqual(first, fixture[-1].read_bytes()) + self.assertTrue(first.startswith(b"\xfd7zXZ\x00")) + with tarfile.open(fixture[-1]) as archive: + compressed = {m.name: (archive.extractfile(m).read(), m.mode) for m in archive} + self.assertEqual(original, compressed) + def test_media_helper_tamper_is_rejected_against_pinned_archive(self): with tempfile.TemporaryDirectory() as directory: fixture = self.fixture(directory)