From 9ce60997db616c998311d24ded1bdb57e197a181 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 15:37:12 +0200 Subject: [PATCH] Require document helper in Linux headless packages and verify Unix worker --- docs/BUILDING.md | 7 +++++ docs/EXTRACTION-ISOLATION.md | 15 +++++++++++ tools/package_linux.py | 13 ++++++++- tools/test_package_linux.py | 51 ++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 9244875..c74949d 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -304,3 +304,10 @@ it with `python tools/stage_extraction_worker.py` before desktop development or release packaging. Release builds reject a missing helper. Headless deployments place the platform helper beside the CLI executable. Parser failure retains filename search and exposes an extraction error; there is no in-process fallback. + +For Linux headless archive preparation, stage the extraction helper into the +same directory passed to `package_linux.py --runtime`: + +```sh +python tools/stage_extraction_worker.py --destination .searchi/runtime/auto +``` diff --git a/docs/EXTRACTION-ISOLATION.md b/docs/EXTRACTION-ISOLATION.md index e033a3c..acb5d1f 100644 --- a/docs/EXTRACTION-ISOLATION.md +++ b/docs/EXTRACTION-ISOLATION.md @@ -178,3 +178,18 @@ its actual entries. The helper extracted from that archive passed genuine PDF indexing through the normal adapter (1.76 s). Archive SHA-256: `760259bbf07ed076857f0e47c0b536eb352e4d00973701bfe115d95f41989daf`. This archive is local evidence; a new installer has not been published. + +## Linux headless consistency + +The Linux packager now requires a current verified helper/inventory in its +runtime directory, checks x86-64 ELF/system dependency closure and executable +archive mode. Nineteen package tests passed normally and with Python `-O`. +Missing/stale/tampered helpers and unsupported dependencies preserve output. + +An immutable source snapshot at `18ce700` executed four actual worker/parent +integration tests under Ubuntu WSL, CPU only: real DOCX, corrupt PDF, exact +source digest, cancellation distinction and malformed protocol rejection. +Unix resource setup executed; adversarial exhaustion was not tested. The initial +WSL nested-virtualization warning did not prevent the terminal passing Cargo run. +The public Linux archive remains the earlier release; updated archive build and +relocated index checks remain open. diff --git a/tools/package_linux.py b/tools/package_linux.py index 611e911..c1cbf60 100644 --- a/tools/package_linux.py +++ b/tools/package_linux.py @@ -17,6 +17,7 @@ import subprocess import tarfile import tempfile from embeddinggemma2.runtime_provenance import verify as verify_provenance +from stage_extraction_worker import verify as verify_extraction ROOT = Path(__file__).resolve().parents[1] MAX_FILE = 256 * 1024 * 1024 @@ -179,6 +180,16 @@ def package(root, executable, runtime, media, output, llama_license=None, compre regular(executable, executable.parent) elf_x64(executable) expected = runtime_files(runtime, root) + extraction = verify_extraction(runtime, root, "linux", "x86_64") + helper = runtime / "searchi-extract-worker" + regular(helper, runtime) + regular(runtime / "extraction-worker.json", runtime) + elf_x64(helper) + helper_metadata = dynamic(helper) + if (helper_metadata.get("RPATH") or helper_metadata.get("RUNPATH") + or set(helper_metadata.get("NEEDED", [])) - SYSTEM): + raise ValueError("Extraction helper has unexpected native dependency or search path") + expected.update(extraction) cli_metadata = dynamic(executable) if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH") or set(cli_metadata.get("NEEDED", [])) - SYSTEM): @@ -214,7 +225,7 @@ def package(root, executable, runtime, media, output, llama_license=None, compre checksum = hashlib.sha256(data).hexdigest() if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]: raise ValueError(f"Source changed during packaging: {name}") - mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644 + mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644 add(archive, name, data, mode) records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)}) instructions = ("searchi — Find what matters. Locally.\n\n" diff --git a/tools/test_package_linux.py b/tools/test_package_linux.py index 9dca311..7d06152 100644 --- a/tools/test_package_linux.py +++ b/tools/test_package_linux.py @@ -10,6 +10,7 @@ from unittest import mock import package_linux as pack from embeddinggemma2 import runtime_provenance as provenance +import stage_extraction_worker as extraction def elf(path, machine=62): @@ -21,12 +22,57 @@ def elf(path, machine=62): class LinuxPackageTests(unittest.TestCase): + def test_extraction_missing_stale_tampered_or_wrong_architecture_preserves_output(self): + for mutation in ("missing", "inventory_missing", "stale", "tampered", "architecture"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + fixture = self.fixture(directory) + root, _, runtime, _, output = fixture + output.write_bytes(b"previous archive") + helper = runtime / "searchi-extract-worker" + if mutation == "missing": + helper.unlink() + elif mutation == "inventory_missing": + (runtime / "extraction-worker.json").unlink() + elif mutation == "stale": + source = root / "crates/searchi-extract-worker/src/main.rs" + source.write_bytes(source.read_bytes() + b"\n// changed\n") + elif mutation == "tampered": + helper.write_bytes(helper.read_bytes() + b"changed") + else: + elf(helper, machine=183) + inventory = runtime / "extraction-worker.json" + manifest = json.loads(inventory.read_text()) + manifest["file"] = extraction.identity(helper)[2] + inventory.write_text(json.dumps(manifest)) + with self.assertRaises(ValueError): + self.run_package(fixture) + self.assertEqual(output.read_bytes(), b"previous archive") + self.assertEqual(list(output.parent.glob("*.partial")), []) + + def test_extraction_dependency_closure_and_search_paths_preserve_output(self): + for metadata in ({"NEEDED": ["libunpackaged.so.1"]}, + {"NEEDED": ["libvulkan.so.1"]}, + {"RUNPATH": ["$ORIGIN"]}, {"RPATH": ["/build/lib"]}): + with self.subTest(metadata=metadata), tempfile.TemporaryDirectory() as directory: + fixture = self.fixture(directory) + fixture[-1].write_bytes(b"previous archive") + def dependencies(path): + return metadata if path.name == "searchi-extract-worker" else self.metadata(path) + with mock.patch.object(pack, "dynamic", side_effect=dependencies), self.assertRaisesRegex(ValueError, "Extraction helper"): + pack.package(*fixture) + self.assertEqual(fixture[-1].read_bytes(), b"previous archive") + self.assertEqual(list(fixture[-1].parent.glob("*.partial")), []) + def fixture(self, directory): root = Path(directory) for name in provenance.INPUTS: path = root / name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes((provenance.ROOT / name).read_bytes()) + for name in extraction.fingerprint()["inputs"]: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((extraction.ROOT / name).read_bytes()) runtime, media = root / "runtime", root / "media" executable = root / "build/searchi" elf(executable) @@ -36,6 +82,9 @@ class LinuxPackageTests(unittest.TestCase): license.write_text("llama MIT notice") names = ["libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0", "libggml-cpu-x64.so", "libggml-vulkan.so"] elf(runtime / "searchi-gemma-worker") + helper = root / "build/searchi-extract-worker" + elf(helper) + extraction.stage(helper, extraction.fingerprint(root), runtime, root) records = [] for name in names: elf(runtime / name) @@ -91,6 +140,8 @@ class LinuxPackageTests(unittest.TestCase): self.assertEqual(member.uid, 0) self.assertEqual(member.mtime, 0) self.assertEqual(archive.getmember("searchi").mode, 0o755) + self.assertEqual(archive.getmember("searchi-extract-worker").mode, 0o755) + self.assertEqual(archive.getmember("extraction-worker.json").mode, 0o644) self.assertEqual(archive.getmember("media-runtime/bin/ffmpeg").mode, 0o755) self.assertEqual(archive.getmember("LICENSE.txt").mode, 0o644)