Require document helper in Linux headless packages and verify Unix worker

This commit is contained in:
karl committed 2026-10-13 15:37:12 +02:00
1 parent 18ce700ba9
commit 9ce60997db
4 files changed
+85 -1

No files matched your search

+7
View File
@@ -304,3 +304,10 @@ it with `python tools/stage_extraction_worker.py` before desktop development or
release packaging. Release builds reject a missing helper. Headless deployments release packaging. Release builds reject a missing helper. Headless deployments
place the platform helper beside the CLI executable. Parser failure retains place the platform helper beside the CLI executable. Parser failure retains
filename search and exposes an extraction error; there is no in-process fallback. filename search and exposes an extraction error; there is no in-process fallback.
For Linux headless archive preparation, stage the extraction helper into the
same directory passed to `package_linux.py --runtime`:
```sh
python tools/stage_extraction_worker.py --destination .searchi/runtime/auto
```
+15
View File
@@ -178,3 +178,18 @@ its actual entries. The helper extracted from that archive passed genuine PDF
indexing through the normal adapter (1.76 s). Archive SHA-256: indexing through the normal adapter (1.76 s). Archive SHA-256:
`760259bbf07ed076857f0e47c0b536eb352e4d00973701bfe115d95f41989daf`. `760259bbf07ed076857f0e47c0b536eb352e4d00973701bfe115d95f41989daf`.
This archive is local evidence; a new installer has not been published. This archive is local evidence; a new installer has not been published.
## Linux headless consistency
The Linux packager now requires a current verified helper/inventory in its
runtime directory, checks x86-64 ELF/system dependency closure and executable
archive mode. Nineteen package tests passed normally and with Python `-O`.
Missing/stale/tampered helpers and unsupported dependencies preserve output.
An immutable source snapshot at `18ce700` executed four actual worker/parent
integration tests under Ubuntu WSL, CPU only: real DOCX, corrupt PDF, exact
source digest, cancellation distinction and malformed protocol rejection.
Unix resource setup executed; adversarial exhaustion was not tested. The initial
WSL nested-virtualization warning did not prevent the terminal passing Cargo run.
The public Linux archive remains the earlier release; updated archive build and
relocated index checks remain open.
+12 -1
View File
@@ -17,6 +17,7 @@ import subprocess
import tarfile import tarfile
import tempfile import tempfile
from embeddinggemma2.runtime_provenance import verify as verify_provenance from embeddinggemma2.runtime_provenance import verify as verify_provenance
from stage_extraction_worker import verify as verify_extraction
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
MAX_FILE = 256 * 1024 * 1024 MAX_FILE = 256 * 1024 * 1024
@@ -179,6 +180,16 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
regular(executable, executable.parent) regular(executable, executable.parent)
elf_x64(executable) elf_x64(executable)
expected = runtime_files(runtime, root) expected = runtime_files(runtime, root)
extraction = verify_extraction(runtime, root, "linux", "x86_64")
helper = runtime / "searchi-extract-worker"
regular(helper, runtime)
regular(runtime / "extraction-worker.json", runtime)
elf_x64(helper)
helper_metadata = dynamic(helper)
if (helper_metadata.get("RPATH") or helper_metadata.get("RUNPATH")
or set(helper_metadata.get("NEEDED", [])) - SYSTEM):
raise ValueError("Extraction helper has unexpected native dependency or search path")
expected.update(extraction)
cli_metadata = dynamic(executable) cli_metadata = dynamic(executable)
if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH") if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH")
or set(cli_metadata.get("NEEDED", [])) - SYSTEM): or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
@@ -214,7 +225,7 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
checksum = hashlib.sha256(data).hexdigest() checksum = hashlib.sha256(data).hexdigest()
if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]: if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]:
raise ValueError(f"Source changed during packaging: {name}") raise ValueError(f"Source changed during packaging: {name}")
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644 mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
add(archive, name, data, mode) add(archive, name, data, mode)
records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)}) records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)})
instructions = ("searchi — Find what matters. Locally.\n\n" instructions = ("searchi — Find what matters. Locally.\n\n"
+51
View File
@@ -10,6 +10,7 @@ from unittest import mock
import package_linux as pack import package_linux as pack
from embeddinggemma2 import runtime_provenance as provenance from embeddinggemma2 import runtime_provenance as provenance
import stage_extraction_worker as extraction
def elf(path, machine=62): def elf(path, machine=62):
@@ -21,12 +22,57 @@ def elf(path, machine=62):
class LinuxPackageTests(unittest.TestCase): class LinuxPackageTests(unittest.TestCase):
def test_extraction_missing_stale_tampered_or_wrong_architecture_preserves_output(self):
for mutation in ("missing", "inventory_missing", "stale", "tampered", "architecture"):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
fixture = self.fixture(directory)
root, _, runtime, _, output = fixture
output.write_bytes(b"previous archive")
helper = runtime / "searchi-extract-worker"
if mutation == "missing":
helper.unlink()
elif mutation == "inventory_missing":
(runtime / "extraction-worker.json").unlink()
elif mutation == "stale":
source = root / "crates/searchi-extract-worker/src/main.rs"
source.write_bytes(source.read_bytes() + b"\n// changed\n")
elif mutation == "tampered":
helper.write_bytes(helper.read_bytes() + b"changed")
else:
elf(helper, machine=183)
inventory = runtime / "extraction-worker.json"
manifest = json.loads(inventory.read_text())
manifest["file"] = extraction.identity(helper)[2]
inventory.write_text(json.dumps(manifest))
with self.assertRaises(ValueError):
self.run_package(fixture)
self.assertEqual(output.read_bytes(), b"previous archive")
self.assertEqual(list(output.parent.glob("*.partial")), [])
def test_extraction_dependency_closure_and_search_paths_preserve_output(self):
for metadata in ({"NEEDED": ["libunpackaged.so.1"]},
{"NEEDED": ["libvulkan.so.1"]},
{"RUNPATH": ["$ORIGIN"]}, {"RPATH": ["/build/lib"]}):
with self.subTest(metadata=metadata), tempfile.TemporaryDirectory() as directory:
fixture = self.fixture(directory)
fixture[-1].write_bytes(b"previous archive")
def dependencies(path):
return metadata if path.name == "searchi-extract-worker" else self.metadata(path)
with mock.patch.object(pack, "dynamic", side_effect=dependencies), self.assertRaisesRegex(ValueError, "Extraction helper"):
pack.package(*fixture)
self.assertEqual(fixture[-1].read_bytes(), b"previous archive")
self.assertEqual(list(fixture[-1].parent.glob("*.partial")), [])
def fixture(self, directory): def fixture(self, directory):
root = Path(directory) root = Path(directory)
for name in provenance.INPUTS: for name in provenance.INPUTS:
path = root / name path = root / name
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((provenance.ROOT / name).read_bytes()) path.write_bytes((provenance.ROOT / name).read_bytes())
for name in extraction.fingerprint()["inputs"]:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((extraction.ROOT / name).read_bytes())
runtime, media = root / "runtime", root / "media" runtime, media = root / "runtime", root / "media"
executable = root / "build/searchi" executable = root / "build/searchi"
elf(executable) elf(executable)
@@ -36,6 +82,9 @@ class LinuxPackageTests(unittest.TestCase):
license.write_text("llama MIT notice") license.write_text("llama MIT notice")
names = ["libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0", "libggml-cpu-x64.so", "libggml-vulkan.so"] names = ["libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0", "libggml-cpu-x64.so", "libggml-vulkan.so"]
elf(runtime / "searchi-gemma-worker") elf(runtime / "searchi-gemma-worker")
helper = root / "build/searchi-extract-worker"
elf(helper)
extraction.stage(helper, extraction.fingerprint(root), runtime, root)
records = [] records = []
for name in names: for name in names:
elf(runtime / name) elf(runtime / name)
@@ -91,6 +140,8 @@ class LinuxPackageTests(unittest.TestCase):
self.assertEqual(member.uid, 0) self.assertEqual(member.uid, 0)
self.assertEqual(member.mtime, 0) self.assertEqual(member.mtime, 0)
self.assertEqual(archive.getmember("searchi").mode, 0o755) self.assertEqual(archive.getmember("searchi").mode, 0o755)
self.assertEqual(archive.getmember("searchi-extract-worker").mode, 0o755)
self.assertEqual(archive.getmember("extraction-worker.json").mode, 0o644)
self.assertEqual(archive.getmember("media-runtime/bin/ffmpeg").mode, 0o755) self.assertEqual(archive.getmember("media-runtime/bin/ffmpeg").mode, 0o755)
self.assertEqual(archive.getmember("LICENSE.txt").mode, 0o644) self.assertEqual(archive.getmember("LICENSE.txt").mode, 0o644)