Stage and require verified extraction helper in Windows release packages

tools/package_windows.py

@@ -14,6 +14,7 @@ import stat
 import tempfile
 import zipfile
 from embeddinggemma2.runtime_provenance import verify as verify_provenance
+from stage_extraction_worker import verify as verify_extraction
 
 ROOT = Path(__file__).resolve().parents[1]
 CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
@@ -146,6 +147,7 @@ def package(root, executable, output, cli=None):
             raise ValueError(f"Runtime size mismatch: {name}")
     media = media_provenance(root, resources, architecture)
     expected.update(media)
+    expected.update(verify_extraction(resources, root, "windows", architecture))
     selected = [(executable, "searchi.exe"), (root / "LICENSE", "LICENSE.txt")]
     if cli is not None:
         if cli.is_symlink() or not cli.is_file() or not 0 < cli.stat().st_size <= MAX_MEMBER:

tools/stage_extraction_worker.py

@@ -0,0 +1,135 @@
+"""Build and stage the isolated document helper; no model or GPU execution."""
+import argparse
+import hashlib
+import json
+import os
+import platform as host_platform
+from pathlib import Path
+import struct
+import subprocess
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[1]
+MAX_BINARY = 64 * 1024 * 1024
+MAX_MANIFEST = 65536
+
+
+def fingerprint(root=ROOT):
+    root = Path(root)
+    names = ['Cargo.toml', 'Cargo.lock']
+    for crate in ('searchi-core', 'searchi-extract-worker', 'searchi-media'):
+        names.append(f'crates/{crate}/Cargo.toml')
+        entry = root / 'crates' / crate / 'src' / ('main.rs' if crate == 'searchi-extract-worker' else 'lib.rs')
+        if not entry.is_file():
+            raise ValueError('Required extraction build source missing')
+        names.extend(path.relative_to(root).as_posix() for path in sorted((root / 'crates' / crate / 'src').rglob('*.rs')))
+    if len(names) < 6 or len(names) > 256:
+        raise ValueError('Extraction build inputs missing or excessive')
+    records = {}
+    for name in names:
+        path = root / name
+        if path.is_symlink():
+            raise ValueError('Linked extraction build input')
+        with path.open("rb") as stream:
+            data = stream.read(2 * 1024 * 1024 + 1)
+        if not data or len(data) > 2 * 1024 * 1024:
+            raise ValueError('Extraction build input empty or oversized')
+        records[name] = hashlib.sha256(data.replace(b'\r\n', b'\n')).hexdigest()
+    return {'schema': 1, 'normalization': 'CRLF-to-LF build inputs only', 'inputs': records}
+
+
+def identity(path):
+    path = Path(path)
+    if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_BINARY:
+        raise ValueError('Extraction helper missing, linked or oversized')
+    with path.open('rb') as stream:
+        header = stream.read(64)
+        if header[:2] == b'MZ' and len(header) == 64:
+            offset = struct.unpack_from('<I', header, 60)[0]
+            if offset > path.stat().st_size - 6:
+                raise ValueError('Invalid extraction PE header')
+            stream.seek(offset)
+            signature = stream.read(6)
+            machine = struct.unpack_from('<H', signature, 4)[0] if signature[:4] == b'PE\0\0' else 0
+            arch = {0x8664: 'x86_64', 0xAA64: 'aarch64'}.get(machine)
+            platform = 'windows'
+        elif header[:6] == b'\x7fELF\x02\x01':
+            machine = struct.unpack_from('<H', header, 18)[0]
+            arch = {62: 'x86_64', 183: 'aarch64'}.get(machine)
+            platform = 'linux'
+        else:
+            raise ValueError('Unsupported extraction executable format')
+        if arch is None:
+            raise ValueError('Unsupported extraction architecture')
+        stream.seek(0)
+        checksum = hashlib.file_digest(stream, 'sha256').hexdigest()
+    return platform, arch, {'bytes': path.stat().st_size, 'sha256': checksum}
+
+
+def verify(resources, root=ROOT, platform='windows', architecture='x86_64'):
+    resources = Path(resources)
+    manifest_path = resources / 'extraction-worker.json'
+    if manifest_path.is_symlink() or not manifest_path.is_file() or manifest_path.stat().st_size > MAX_MANIFEST:
+        raise ValueError('Extraction worker inventory missing or oversized')
+    data = manifest_path.read_bytes()
+    manifest = json.loads(data)
+    name = 'searchi-extract-worker.exe' if platform == 'windows' else 'searchi-extract-worker'
+    if not isinstance(manifest, dict) or manifest.get('schema') != 1 or manifest.get('build_provenance') != fingerprint(root):
+        raise ValueError('Extraction worker provenance missing or stale')
+    if manifest.get('platform') != platform or manifest.get('architecture') != architecture or manifest.get('path') != name:
+        raise ValueError('Extraction worker architecture or path mismatch')
+    detected_platform, detected_arch, record = identity(resources / name)
+    if (detected_platform, detected_arch) != (platform, architecture) or manifest.get('file') != record:
+        raise ValueError('Extraction helper checksum, size or architecture mismatch')
+    return {name: record, manifest_path.name: {'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()}}
+
+
+def stage(binary, provenance, destination, root=ROOT):
+    if provenance != fingerprint(root):
+        raise ValueError('Extraction sources changed during build')
+    platform, architecture, record = identity(binary)
+    name = 'searchi-extract-worker.exe' if platform == 'windows' else 'searchi-extract-worker'
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    with Path(binary).open('rb') as stream:
+        binary_bytes = stream.read(MAX_BINARY + 1)
+    if len(binary_bytes) != record['bytes'] or hashlib.sha256(binary_bytes).hexdigest() != record['sha256']:
+        raise ValueError('Extraction binary changed while staging')
+    manifest = {'schema': 1, 'platform': platform, 'architecture': architecture,
+                'path': name, 'file': record, 'build_provenance': provenance}
+    # Only these two owned files are replaced. Commit the inventory last; an
+    # interrupted copy cannot pass verification against a previous inventory.
+    for target, data in ((name, binary_bytes), ('extraction-worker.json', json.dumps(manifest, indent=2).encode())):
+        with tempfile.NamedTemporaryFile(dir=destination, delete=False, suffix='.partial') as stream:
+            temporary = Path(stream.name)
+            stream.write(data)
+        try:
+            if target == name and hashlib.sha256(data).hexdigest() != record['sha256']:
+                raise ValueError('Extraction binary changed while staging')
+            if target == name and platform == 'linux':
+                temporary.chmod(0o755)
+            os.replace(temporary, destination / target)
+        finally:
+            temporary.unlink(missing_ok=True)
+    verify(destination, root, platform, architecture)
+    return manifest
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--destination', type=Path, default=ROOT / 'apps/desktop/src-tauri/resources')
+    parser.add_argument('--verify-only', action='store_true')
+    parser.add_argument('--platform', choices=['windows', 'linux'], default='windows' if os.name == 'nt' else 'linux')
+    parser.add_argument('--architecture', choices=['x86_64', 'aarch64'], default='aarch64' if host_platform.machine().lower() in {'aarch64', 'arm64'} else 'x86_64')
+    args = parser.parse_args()
+    if args.verify_only:
+        verify(args.destination, ROOT, args.platform, args.architecture)
+        return
+    before = fingerprint()
+    subprocess.run(['cargo', 'build', '--release', '-p', 'searchi-extract-worker', '--locked'], cwd=ROOT, check=True)
+    suffix = '.exe' if os.name == 'nt' else ''
+    stage(ROOT / 'target/release' / ('searchi-extract-worker' + suffix), before, args.destination)
+
+
+if __name__ == '__main__':
+    main()

tools/test_package_windows.py

@@ -9,6 +9,7 @@ import zipfile
 
 from package_windows import package, CRT_LIBRARIES
 from embeddinggemma2 import runtime_provenance as provenance
+import stage_extraction_worker as extraction
 
 
 def executable(path, machine=0x8664):
@@ -31,6 +32,24 @@ def inventory(resources):
 
 
 class PackageTests(unittest.TestCase):
+    def test_extraction_missing_tampered_stale_preserves_existing_package(self):
+        for mutation in ("missing", "tampered", "stale"):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                root, resources, app, output = self.fixture(directory)
+                output.write_bytes(b"previous package")
+                helper = resources / "searchi-extract-worker.exe"
+                if mutation == "missing":
+                    helper.unlink()
+                elif mutation == "tampered":
+                    helper.write_bytes(helper.read_bytes() + b"changed")
+                else:
+                    source = root / "crates/searchi-extract-worker/src/main.rs"
+                    source.write_bytes(source.read_bytes() + b"\n// changed\n")
+                with self.assertRaises(ValueError):
+                    package(root, app, output)
+                self.assertEqual(output.read_bytes(), b"previous package")
+                self.assertEqual(list(output.parent.glob("*.partial")), [])
+
     def test_optional_cli_manifest_and_architecture(self):
         with tempfile.TemporaryDirectory() as directory:
             root, _, app, output = self.fixture(directory)
@@ -67,6 +86,10 @@ class PackageTests(unittest.TestCase):
             path = root / name
             path.parent.mkdir(parents=True, exist_ok=True)
             path.write_bytes((provenance.ROOT / name).read_bytes())
+        for name in extraction.fingerprint()["inputs"]:
+            path = root / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes((extraction.ROOT / name).read_bytes())
         resources = root / "apps/desktop/src-tauri/resources"
         app = root / "build/app.exe"
         executable(app)
@@ -93,6 +116,9 @@ class PackageTests(unittest.TestCase):
             "sha256": hashlib.sha256(upstream.read_bytes()).hexdigest()}}}))
         (resources / "media-runtime/manifest.json").write_bytes(manifest.read_bytes())
         inventory(resources)
+        helper = root / "build/searchi-extract-worker.exe"
+        executable(helper)
+        extraction.stage(helper, extraction.fingerprint(root), resources, root)
         return root, resources, app, root / "archive.zip"
 
     def test_valid_pe_media_mutation_and_notice_corruption_preserve_previous_archive(self):

tools/test_stage_extraction_worker.py

@@ -0,0 +1,67 @@
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+import stage_extraction_worker as worker
+from test_package_windows import executable
+
+
+class ExtractionStageTests(unittest.TestCase):
+    def fixture(self, directory):
+        root = Path(directory)
+        for name in worker.fingerprint()['inputs']:
+            path = root / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes((worker.ROOT / name).read_bytes())
+        binary = root / 'build/helper.exe'
+        executable(binary)
+        resources = root / 'resources'
+        resources.mkdir()
+        (resources / 'runtime-libraries.json').write_bytes(b'preserve Google')
+        worker.stage(binary, worker.fingerprint(root), resources, root)
+        return root, binary, resources
+
+    def test_valid_staging_preserves_unowned_resources(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root, _, resources = self.fixture(directory)
+            records = worker.verify(resources, root)
+            self.assertEqual(set(records), {'searchi-extract-worker.exe', 'extraction-worker.json'})
+            self.assertEqual((resources / 'runtime-libraries.json').read_bytes(), b'preserve Google')
+
+    def test_missing_tampered_stale_and_wrong_architecture_rejected(self):
+        for mutation in ('missing', 'tampered', 'stale', 'architecture'):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                root, binary, resources = self.fixture(directory)
+                helper = resources / 'searchi-extract-worker.exe'
+                if mutation == 'missing':
+                    helper.unlink()
+                elif mutation == 'tampered':
+                    helper.write_bytes(helper.read_bytes() + b'changed')
+                elif mutation == 'stale':
+                    source = root / 'crates/searchi-extract-worker/src/main.rs'
+                    source.write_bytes(source.read_bytes() + b'\n// changed\n')
+                else:
+                    executable(helper, 0xAA64)
+                with self.assertRaises(ValueError):
+                    worker.verify(resources, root)
+
+    def test_build_mutation_does_not_replace_staged_helper(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root, binary, resources = self.fixture(directory)
+            before = worker.fingerprint(root)
+            source = root / 'Cargo.toml'
+            source.write_bytes(source.read_bytes() + b'\n')
+            with self.assertRaisesRegex(ValueError, 'changed during build'):
+                worker.stage(binary, before, resources, root)
+            self.assertEqual((resources / 'searchi-extract-worker.exe').read_bytes(), binary.read_bytes())
+
+    def test_failed_build_never_stages_existing_binary(self):
+        with patch.object(worker.subprocess, 'run', side_effect=RuntimeError('cargo failed')), patch.object(worker, 'stage') as stage, patch('sys.argv', ['stage_extraction_worker.py']):
+            with self.assertRaises(RuntimeError):
+                worker.main()
+            stage.assert_not_called()
+
+
+if __name__ == '__main__':
+    unittest.main()