Require document helper in Linux headless packages and verify Unix worker

This commit is contained in:
karl committed 2026-10-13 15:37:12 +02:00
1 parent 18ce700ba9
commit 9ce60997db
4 files changed
+85 -1

No files matched your search

+7
View File
@@ -304,3 +304,10 @@ it with `python tools/stage_extraction_worker.py` before desktop development or
release packaging. Release builds reject a missing helper. Headless deployments
place the platform helper beside the CLI executable. Parser failure retains
filename search and exposes an extraction error; there is no in-process fallback.
For Linux headless archive preparation, stage the extraction helper into the
same directory passed to `package_linux.py --runtime`:
```sh
python tools/stage_extraction_worker.py --destination .searchi/runtime/auto
```
+15
View File
@@ -178,3 +178,18 @@ its actual entries. The helper extracted from that archive passed genuine PDF
indexing through the normal adapter (1.76 s). Archive SHA-256:
`760259bbf07ed076857f0e47c0b536eb352e4d00973701bfe115d95f41989daf`.
This archive is local evidence; a new installer has not been published.
## Linux headless consistency
The Linux packager now requires a current verified helper/inventory in its
runtime directory, checks x86-64 ELF/system dependency closure and executable
archive mode. Nineteen package tests passed normally and with Python `-O`.
Missing/stale/tampered helpers and unsupported dependencies preserve output.
An immutable source snapshot at `18ce700` executed four actual worker/parent
integration tests under Ubuntu WSL, CPU only: real DOCX, corrupt PDF, exact
source digest, cancellation distinction and malformed protocol rejection.
Unix resource setup executed; adversarial exhaustion was not tested. The initial
WSL nested-virtualization warning did not prevent the terminal passing Cargo run.
The public Linux archive remains the earlier release; updated archive build and
relocated index checks remain open.
+12 -1
View File
@@ -17,6 +17,7 @@ import subprocess
import tarfile
import tempfile
from embeddinggemma2.runtime_provenance import verify as verify_provenance
from stage_extraction_worker import verify as verify_extraction
ROOT = Path(__file__).resolve().parents[1]
MAX_FILE = 256 * 1024 * 1024
@@ -179,6 +180,16 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
regular(executable, executable.parent)
elf_x64(executable)
expected = runtime_files(runtime, root)
extraction = verify_extraction(runtime, root, "linux", "x86_64")
helper = runtime / "searchi-extract-worker"
regular(helper, runtime)
regular(runtime / "extraction-worker.json", runtime)
elf_x64(helper)
helper_metadata = dynamic(helper)
if (helper_metadata.get("RPATH") or helper_metadata.get("RUNPATH")
or set(helper_metadata.get("NEEDED", [])) - SYSTEM):
raise ValueError("Extraction helper has unexpected native dependency or search path")
expected.update(extraction)
cli_metadata = dynamic(executable)
if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH")
or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
@@ -214,7 +225,7 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
checksum = hashlib.sha256(data).hexdigest()
if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]:
raise ValueError(f"Source changed during packaging: {name}")
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
add(archive, name, data, mode)
records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)})
instructions = ("searchi — Find what matters. Locally.\n\n"
+51
View File
@@ -10,6 +10,7 @@ from unittest import mock
import package_linux as pack
from embeddinggemma2 import runtime_provenance as provenance
import stage_extraction_worker as extraction
def elf(path, machine=62):
@@ -21,12 +22,57 @@ def elf(path, machine=62):
class LinuxPackageTests(unittest.TestCase):
def test_extraction_missing_stale_tampered_or_wrong_architecture_preserves_output(self):
for mutation in ("missing", "inventory_missing", "stale", "tampered", "architecture"):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
fixture = self.fixture(directory)
root, _, runtime, _, output = fixture
output.write_bytes(b"previous archive")
helper = runtime / "searchi-extract-worker"
if mutation == "missing":
helper.unlink()
elif mutation == "inventory_missing":
(runtime / "extraction-worker.json").unlink()
elif mutation == "stale":
source = root / "crates/searchi-extract-worker/src/main.rs"
source.write_bytes(source.read_bytes() + b"\n// changed\n")
elif mutation == "tampered":
helper.write_bytes(helper.read_bytes() + b"changed")
else:
elf(helper, machine=183)
inventory = runtime / "extraction-worker.json"
manifest = json.loads(inventory.read_text())
manifest["file"] = extraction.identity(helper)[2]
inventory.write_text(json.dumps(manifest))
with self.assertRaises(ValueError):
self.run_package(fixture)
self.assertEqual(output.read_bytes(), b"previous archive")
self.assertEqual(list(output.parent.glob("*.partial")), [])
def test_extraction_dependency_closure_and_search_paths_preserve_output(self):
for metadata in ({"NEEDED": ["libunpackaged.so.1"]},
{"NEEDED": ["libvulkan.so.1"]},
{"RUNPATH": ["$ORIGIN"]}, {"RPATH": ["/build/lib"]}):
with self.subTest(metadata=metadata), tempfile.TemporaryDirectory() as directory:
fixture = self.fixture(directory)
fixture[-1].write_bytes(b"previous archive")
def dependencies(path):
return metadata if path.name == "searchi-extract-worker" else self.metadata(path)
with mock.patch.object(pack, "dynamic", side_effect=dependencies), self.assertRaisesRegex(ValueError, "Extraction helper"):
pack.package(*fixture)
self.assertEqual(fixture[-1].read_bytes(), b"previous archive")
self.assertEqual(list(fixture[-1].parent.glob("*.partial")), [])
def fixture(self, directory):
root = Path(directory)
for name in provenance.INPUTS:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((provenance.ROOT / name).read_bytes())
for name in extraction.fingerprint()["inputs"]:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes((extraction.ROOT / name).read_bytes())
runtime, media = root / "runtime", root / "media"
executable = root / "build/searchi"
elf(executable)
@@ -36,6 +82,9 @@ class LinuxPackageTests(unittest.TestCase):
license.write_text("llama MIT notice")
names = ["libggml.so.0", "libggml-base.so.0", "libllama.so.0", "libmtmd.so.0", "libggml-cpu-x64.so", "libggml-vulkan.so"]
elf(runtime / "searchi-gemma-worker")
helper = root / "build/searchi-extract-worker"
elf(helper)
extraction.stage(helper, extraction.fingerprint(root), runtime, root)
records = []
for name in names:
elf(runtime / name)
@@ -91,6 +140,8 @@ class LinuxPackageTests(unittest.TestCase):
self.assertEqual(member.uid, 0)
self.assertEqual(member.mtime, 0)
self.assertEqual(archive.getmember("searchi").mode, 0o755)
self.assertEqual(archive.getmember("searchi-extract-worker").mode, 0o755)
self.assertEqual(archive.getmember("extraction-worker.json").mode, 0o644)
self.assertEqual(archive.getmember("media-runtime/bin/ffmpeg").mode, 0o755)
self.assertEqual(archive.getmember("LICENSE.txt").mode, 0o644)