From e250577740f0da1d54d829714f10d4596280a03f Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 15:23:24 +0200 Subject: [PATCH] Stage and require verified extraction helper in Windows release packages --- .gitignore | 3 + apps/desktop/src-tauri/build.rs | 43 ++++++++ docs/BUILDING.md | 7 ++ docs/EXTRACTION-ISOLATION.md | 21 ++++ tools/package_windows.py | 2 + tools/stage_extraction_worker.py | 135 ++++++++++++++++++++++++++ tools/test_package_windows.py | 26 +++++ tools/test_stage_extraction_worker.py | 67 +++++++++++++ 8 files changed, 304 insertions(+) create mode 100644 tools/stage_extraction_worker.py create mode 100644 tools/test_stage_extraction_worker.py diff --git a/.gitignore b/.gitignore index 10ec936..955705f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ __pycache__/ /apps/desktop/src-tauri/resources/media-runtime/bin/ /apps/desktop/src-tauri/icons/android/ /apps/desktop/src-tauri/icons/ios/ + +/apps/desktop/src-tauri/resources/searchi-extract-worker* +/apps/desktop/src-tauri/resources/extraction-worker.json diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rs index d860e1e..58fcef1 100644 --- a/apps/desktop/src-tauri/build.rs +++ b/apps/desktop/src-tauri/build.rs @@ -1,3 +1,46 @@ fn main() { + let target = std::env::var("CARGO_CFG_TARGET_OS").expect("target OS"); + let name = if target == "windows" { + "searchi-extract-worker.exe" + } else { + "searchi-extract-worker" + }; + let helper = std::path::Path::new("resources").join(name); + println!("cargo:rerun-if-changed={}", helper.display()); + println!("cargo:rerun-if-changed=resources/extraction-worker.json"); + println!("cargo:rerun-if-changed=../../../tools/stage_extraction_worker.py"); + println!("cargo:rerun-if-changed=../../../crates/searchi-core/src"); + println!("cargo:rerun-if-changed=../../../crates/searchi-media/src"); + println!("cargo:rerun-if-changed=../../../crates/searchi-extract-worker/src"); + println!("cargo:rerun-if-changed=../../../Cargo.lock"); + println!("cargo:rerun-if-changed=../../../Cargo.toml"); + if std::env::var("PROFILE").as_deref() == Ok("release") { + let metadata = std::fs::symlink_metadata(&helper).unwrap_or_else(|_| { + panic!("Missing packaged document helper. Run tools/stage_extraction_worker.py before building a release.") + }); + assert!( + metadata.file_type().is_file() + && metadata.len() > 0 + && metadata.len() <= 64 * 1024 * 1024, + "Packaged document helper must be a regular, nonempty file under 64 MiB" + ); + } + if std::env::var("PROFILE").as_deref() == Ok("release") + && matches!(target.as_str(), "windows" | "linux") + { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let status = std::process::Command::new("python") + .arg(root.join("tools/stage_extraction_worker.py")) + .args(["--verify-only", "--platform", &target, "--architecture"]) + .arg(std::env::var("CARGO_CFG_TARGET_ARCH").expect("target architecture")) + .arg("--destination") + .arg(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources")) + .status() + .expect("Python is required to verify packaged extraction worker provenance"); + assert!( + status.success(), + "Packaged extraction worker verification failed; rebuild and stage it" + ); + } tauri_build::build() } diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 4272787..9244875 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -27,6 +27,7 @@ python -m pip install cmake python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2 python tools/embeddinggemma2/stage_runtime.py --backend cpu python tools/media-runtime/provision.py --stage +python tools/stage_extraction_worker.py python tools/embeddinggemma2/download.py $env:SEARCHI_FORCE_CPU = '1' $env:SEARCHI_GEMMA_MODEL_DIR = (Resolve-Path .searchi/models/embeddinggemma2).Path @@ -297,3 +298,9 @@ Windows and Linux x86-64 startup and CPU decoding have executed here; other architectures and macOS decoder packaging remain gates. See [media runtime provenance](../tools/media-runtime/README.md) and [measured media limitations](MEDIA-QUALITY.md). + +PDF/DOCX indexing requires the bundled `searchi-extract-worker`. Build and stage +it with `python tools/stage_extraction_worker.py` before desktop development or +release packaging. Release builds reject a missing helper. Headless deployments +place the platform helper beside the CLI executable. Parser failure retains +filename search and exposes an extraction error; there is no in-process fallback. diff --git a/docs/EXTRACTION-ISOLATION.md b/docs/EXTRACTION-ISOLATION.md index bc31020..12b5368 100644 --- a/docs/EXTRACTION-ISOLATION.md +++ b/docs/EXTRACTION-ISOLATION.md @@ -150,3 +150,24 @@ installer contains this helper. Do not publish current source without it. Desktop integration validation: 46 native tests passed on Windows with CPU-only settings. The desktop lockfile now records the shared process adapter dependencies. This checks compiled integration, not installed helper discovery or native UI interaction. + +The optimized Windows helper built successfully and was copied into an owned +validation directory outside `target`. The normal-adapter genuine PDF test +passed against that relocated release helper (one test, 1.11 s). Its SHA-256 is +`983156ddb96de1b4fb307564c1b882ffb75300e05f03f705ee8e2e668ff96ae1`. +This is helper relocation evidence, not a finished installer check. + +## Windows release staging + +`tools/stage_extraction_worker.py` performs a locked optimized Cargo build, +compares bounded core/worker/media build inputs before and after it, and stages +only the helper and `extraction-worker.json`. Binary hash, size, architecture +and source provenance are verified. Unrelated runtime resources are retained. +Windows archive packaging requires the verified pair before output replacement; +release desktop builds run the same verification before bundling. + +The real Windows build/stage/verification passed. Twenty-two tooling tests +passed normally and under Python `-O`, including missing, stale, tampered and +architecture-mismatched helpers plus failed builds and preserved unrelated files. +Linux headless packaging and native macOS helper staging remain open. A complete +updated Windows archive/installer has not yet been verified or published. diff --git a/tools/package_windows.py b/tools/package_windows.py index b5fb9cc..910825f 100644 --- a/tools/package_windows.py +++ b/tools/package_windows.py @@ -14,6 +14,7 @@ import stat import tempfile import zipfile from embeddinggemma2.runtime_provenance import verify as verify_provenance +from stage_extraction_worker import verify as verify_extraction ROOT = Path(__file__).resolve().parents[1] CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"} @@ -146,6 +147,7 @@ def package(root, executable, output, cli=None): raise ValueError(f"Runtime size mismatch: {name}") media = media_provenance(root, resources, architecture) expected.update(media) + expected.update(verify_extraction(resources, root, "windows", architecture)) selected = [(executable, "searchi.exe"), (root / "LICENSE", "LICENSE.txt")] if cli is not None: if cli.is_symlink() or not cli.is_file() or not 0 < cli.stat().st_size <= MAX_MEMBER: diff --git a/tools/stage_extraction_worker.py b/tools/stage_extraction_worker.py new file mode 100644 index 0000000..e817d0f --- /dev/null +++ b/tools/stage_extraction_worker.py @@ -0,0 +1,135 @@ +"""Build and stage the isolated document helper; no model or GPU execution.""" +import argparse +import hashlib +import json +import os +import platform as host_platform +from pathlib import Path +import struct +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +MAX_BINARY = 64 * 1024 * 1024 +MAX_MANIFEST = 65536 + + +def fingerprint(root=ROOT): + root = Path(root) + names = ['Cargo.toml', 'Cargo.lock'] + for crate in ('searchi-core', 'searchi-extract-worker', 'searchi-media'): + names.append(f'crates/{crate}/Cargo.toml') + entry = root / 'crates' / crate / 'src' / ('main.rs' if crate == 'searchi-extract-worker' else 'lib.rs') + if not entry.is_file(): + raise ValueError('Required extraction build source missing') + names.extend(path.relative_to(root).as_posix() for path in sorted((root / 'crates' / crate / 'src').rglob('*.rs'))) + if len(names) < 6 or len(names) > 256: + raise ValueError('Extraction build inputs missing or excessive') + records = {} + for name in names: + path = root / name + if path.is_symlink(): + raise ValueError('Linked extraction build input') + with path.open("rb") as stream: + data = stream.read(2 * 1024 * 1024 + 1) + if not data or len(data) > 2 * 1024 * 1024: + raise ValueError('Extraction build input empty or oversized') + records[name] = hashlib.sha256(data.replace(b'\r\n', b'\n')).hexdigest() + return {'schema': 1, 'normalization': 'CRLF-to-LF build inputs only', 'inputs': records} + + +def identity(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_BINARY: + raise ValueError('Extraction helper missing, linked or oversized') + with path.open('rb') as stream: + header = stream.read(64) + if header[:2] == b'MZ' and len(header) == 64: + offset = struct.unpack_from(' path.stat().st_size - 6: + raise ValueError('Invalid extraction PE header') + stream.seek(offset) + signature = stream.read(6) + machine = struct.unpack_from(' MAX_MANIFEST: + raise ValueError('Extraction worker inventory missing or oversized') + data = manifest_path.read_bytes() + manifest = json.loads(data) + name = 'searchi-extract-worker.exe' if platform == 'windows' else 'searchi-extract-worker' + if not isinstance(manifest, dict) or manifest.get('schema') != 1 or manifest.get('build_provenance') != fingerprint(root): + raise ValueError('Extraction worker provenance missing or stale') + if manifest.get('platform') != platform or manifest.get('architecture') != architecture or manifest.get('path') != name: + raise ValueError('Extraction worker architecture or path mismatch') + detected_platform, detected_arch, record = identity(resources / name) + if (detected_platform, detected_arch) != (platform, architecture) or manifest.get('file') != record: + raise ValueError('Extraction helper checksum, size or architecture mismatch') + return {name: record, manifest_path.name: {'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()}} + + +def stage(binary, provenance, destination, root=ROOT): + if provenance != fingerprint(root): + raise ValueError('Extraction sources changed during build') + platform, architecture, record = identity(binary) + name = 'searchi-extract-worker.exe' if platform == 'windows' else 'searchi-extract-worker' + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + with Path(binary).open('rb') as stream: + binary_bytes = stream.read(MAX_BINARY + 1) + if len(binary_bytes) != record['bytes'] or hashlib.sha256(binary_bytes).hexdigest() != record['sha256']: + raise ValueError('Extraction binary changed while staging') + manifest = {'schema': 1, 'platform': platform, 'architecture': architecture, + 'path': name, 'file': record, 'build_provenance': provenance} + # Only these two owned files are replaced. Commit the inventory last; an + # interrupted copy cannot pass verification against a previous inventory. + for target, data in ((name, binary_bytes), ('extraction-worker.json', json.dumps(manifest, indent=2).encode())): + with tempfile.NamedTemporaryFile(dir=destination, delete=False, suffix='.partial') as stream: + temporary = Path(stream.name) + stream.write(data) + try: + if target == name and hashlib.sha256(data).hexdigest() != record['sha256']: + raise ValueError('Extraction binary changed while staging') + if target == name and platform == 'linux': + temporary.chmod(0o755) + os.replace(temporary, destination / target) + finally: + temporary.unlink(missing_ok=True) + verify(destination, root, platform, architecture) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--destination', type=Path, default=ROOT / 'apps/desktop/src-tauri/resources') + parser.add_argument('--verify-only', action='store_true') + parser.add_argument('--platform', choices=['windows', 'linux'], default='windows' if os.name == 'nt' else 'linux') + parser.add_argument('--architecture', choices=['x86_64', 'aarch64'], default='aarch64' if host_platform.machine().lower() in {'aarch64', 'arm64'} else 'x86_64') + args = parser.parse_args() + if args.verify_only: + verify(args.destination, ROOT, args.platform, args.architecture) + return + before = fingerprint() + subprocess.run(['cargo', 'build', '--release', '-p', 'searchi-extract-worker', '--locked'], cwd=ROOT, check=True) + suffix = '.exe' if os.name == 'nt' else '' + stage(ROOT / 'target/release' / ('searchi-extract-worker' + suffix), before, args.destination) + + +if __name__ == '__main__': + main() diff --git a/tools/test_package_windows.py b/tools/test_package_windows.py index 59dc8e7..9daa54f 100644 --- a/tools/test_package_windows.py +++ b/tools/test_package_windows.py @@ -9,6 +9,7 @@ import zipfile from package_windows import package, CRT_LIBRARIES from embeddinggemma2 import runtime_provenance as provenance +import stage_extraction_worker as extraction def executable(path, machine=0x8664): @@ -31,6 +32,24 @@ def inventory(resources): class PackageTests(unittest.TestCase): + def test_extraction_missing_tampered_stale_preserves_existing_package(self): + for mutation in ("missing", "tampered", "stale"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + root, resources, app, output = self.fixture(directory) + output.write_bytes(b"previous package") + helper = resources / "searchi-extract-worker.exe" + if mutation == "missing": + helper.unlink() + elif mutation == "tampered": + helper.write_bytes(helper.read_bytes() + b"changed") + else: + source = root / "crates/searchi-extract-worker/src/main.rs" + source.write_bytes(source.read_bytes() + b"\n// changed\n") + with self.assertRaises(ValueError): + package(root, app, output) + self.assertEqual(output.read_bytes(), b"previous package") + self.assertEqual(list(output.parent.glob("*.partial")), []) + def test_optional_cli_manifest_and_architecture(self): with tempfile.TemporaryDirectory() as directory: root, _, app, output = self.fixture(directory) @@ -67,6 +86,10 @@ class PackageTests(unittest.TestCase): path = root / name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes((provenance.ROOT / name).read_bytes()) + for name in extraction.fingerprint()["inputs"]: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((extraction.ROOT / name).read_bytes()) resources = root / "apps/desktop/src-tauri/resources" app = root / "build/app.exe" executable(app) @@ -93,6 +116,9 @@ class PackageTests(unittest.TestCase): "sha256": hashlib.sha256(upstream.read_bytes()).hexdigest()}}})) (resources / "media-runtime/manifest.json").write_bytes(manifest.read_bytes()) inventory(resources) + helper = root / "build/searchi-extract-worker.exe" + executable(helper) + extraction.stage(helper, extraction.fingerprint(root), resources, root) return root, resources, app, root / "archive.zip" def test_valid_pe_media_mutation_and_notice_corruption_preserve_previous_archive(self): diff --git a/tools/test_stage_extraction_worker.py b/tools/test_stage_extraction_worker.py new file mode 100644 index 0000000..137aecb --- /dev/null +++ b/tools/test_stage_extraction_worker.py @@ -0,0 +1,67 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import stage_extraction_worker as worker +from test_package_windows import executable + + +class ExtractionStageTests(unittest.TestCase): + def fixture(self, directory): + root = Path(directory) + for name in worker.fingerprint()['inputs']: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((worker.ROOT / name).read_bytes()) + binary = root / 'build/helper.exe' + executable(binary) + resources = root / 'resources' + resources.mkdir() + (resources / 'runtime-libraries.json').write_bytes(b'preserve Google') + worker.stage(binary, worker.fingerprint(root), resources, root) + return root, binary, resources + + def test_valid_staging_preserves_unowned_resources(self): + with tempfile.TemporaryDirectory() as directory: + root, _, resources = self.fixture(directory) + records = worker.verify(resources, root) + self.assertEqual(set(records), {'searchi-extract-worker.exe', 'extraction-worker.json'}) + self.assertEqual((resources / 'runtime-libraries.json').read_bytes(), b'preserve Google') + + def test_missing_tampered_stale_and_wrong_architecture_rejected(self): + for mutation in ('missing', 'tampered', 'stale', 'architecture'): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + root, binary, resources = self.fixture(directory) + helper = resources / 'searchi-extract-worker.exe' + if mutation == 'missing': + helper.unlink() + elif mutation == 'tampered': + helper.write_bytes(helper.read_bytes() + b'changed') + elif mutation == 'stale': + source = root / 'crates/searchi-extract-worker/src/main.rs' + source.write_bytes(source.read_bytes() + b'\n// changed\n') + else: + executable(helper, 0xAA64) + with self.assertRaises(ValueError): + worker.verify(resources, root) + + def test_build_mutation_does_not_replace_staged_helper(self): + with tempfile.TemporaryDirectory() as directory: + root, binary, resources = self.fixture(directory) + before = worker.fingerprint(root) + source = root / 'Cargo.toml' + source.write_bytes(source.read_bytes() + b'\n') + with self.assertRaisesRegex(ValueError, 'changed during build'): + worker.stage(binary, before, resources, root) + self.assertEqual((resources / 'searchi-extract-worker.exe').read_bytes(), binary.read_bytes()) + + def test_failed_build_never_stages_existing_binary(self): + with patch.object(worker.subprocess, 'run', side_effect=RuntimeError('cargo failed')), patch.object(worker, 'stage') as stage, patch('sys.argv', ['stage_extraction_worker.py']): + with self.assertRaises(RuntimeError): + worker.main() + stage.assert_not_called() + + +if __name__ == '__main__': + unittest.main()