Harden worker IPC and recoverable runtime staging

tools/embeddinggemma2/stage_runtime.py

@@ -1,16 +1,327 @@
 """Stage the native worker for installers; optional offline model bundle."""
 import argparse
+from contextlib import contextmanager
 import hashlib
 import json
 import os
 import re
 from pathlib import Path
 import shutil
+import stat
+import time
+import uuid
 
 ROOT = Path(__file__).resolve().parents[2]
 CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
 
 
+def _is_link(path):
+    """Reject Windows reparse points, including junctions, on Python 3.11+."""
+    path = Path(path)
+    if path.is_symlink():
+        return True
+    try:
+        attributes = getattr(path.lstat(), "st_file_attributes", 0)
+    except FileNotFoundError:
+        return False
+    return bool(attributes & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
+
+
+def _checked_path(path, parent):
+    """Validate absolute workspace boundaries before any move or recursive delete."""
+    path, parent = Path(path).absolute(), Path(parent).resolve()
+    if _is_link(path) or path.parent.resolve() != parent:
+        raise RuntimeError("Staging path escapes its checked parent or is a link")
+    if path.exists() and path.resolve() != parent / path.name:
+        raise RuntimeError("Staging path resolves outside its checked parent")
+    return path
+
+
+def _check_tree(directory):
+    if _is_link(directory):
+        raise RuntimeError("Resource staging refuses symlinks or junctions")
+    count = 0
+    for parent, directories, files in os.walk(directory, followlinks=False):
+        for name in [*directories, *files]:
+            count += 1
+            if count > 100000:
+                raise RuntimeError("Resource staging exceeds its file-count limit")
+            path = Path(parent) / name
+            if _is_link(path) or not path.resolve().is_relative_to(directory.resolve()):
+                raise RuntimeError("Resource staging refuses symlinks or junctions")
+
+
+def _remove_tree(path, parent):
+    checked = _checked_path(path, parent)
+    if checked.exists():
+        _check_tree(checked)
+        shutil.rmtree(checked)
+
+
+def _move_directory(source, destination, parent):
+    """Bounded retry for transient Windows sharing locks during directory swaps."""
+    for attempt in range(4):
+        source = _checked_path(source, parent)
+        destination = _checked_path(destination, parent)
+        try:
+            os.replace(source, destination)
+            return
+        except PermissionError:
+            if attempt == 3:
+                raise
+            time.sleep(0.05 * (2 ** attempt))
+
+
+def _record(path):
+    path = Path(path)
+    if _is_link(path) or path.absolute() != path.resolve() or not path.is_file():
+        raise RuntimeError(f"Staging source is missing or is a link: {path.name}")
+    with path.open("rb") as stream:
+        return path.stat().st_size, hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def _verified_copy(source, destination, expected=None):
+    source, destination = Path(source), Path(destination)
+    expected = expected or _record(source)
+    if destination.absolute() != destination.resolve():
+        raise RuntimeError("Staging copy target resolves through a link")
+    shutil.copy2(source, destination)
+    if _record(destination) != expected:
+        raise RuntimeError(f"Staging source changed or copy was incomplete: {source.name}")
+    return str(destination)
+
+
+@contextmanager
+def staging_lock(parent):
+    """Kernel-held lock releases after process crashes; PID is informational."""
+    path = _checked_path(parent / ".searchi-stage.lock", parent)
+    with path.open("a+b") as stream:
+        stream.seek(0, os.SEEK_END)
+        if stream.tell() == 0:
+            stream.write(b"\0")
+            stream.flush()
+        stream.seek(0)
+        try:
+            if os.name == "nt":
+                import msvcrt
+                msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+            else:
+                import fcntl
+                fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as error:
+            raise RuntimeError("Another resource staging process holds the lock") from error
+        try:
+            stream.seek(1)
+            stream.truncate()
+            stream.write(f"pid={os.getpid()}".encode())
+            stream.flush()
+            yield
+        finally:
+            stream.seek(0)
+            if os.name == "nt":
+                msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
+
+
+def _journal_paths(destination, state):
+    if (not isinstance(state, dict) or state.get("schema") != 1
+            or state.get("owner") != "searchi-runtime-staging"
+            or not isinstance(state.get("transaction"), str)
+            or not re.fullmatch(r"[0-9a-f]{32}", state["transaction"])
+            or type(state.get("had_destination")) is not bool):
+        raise RuntimeError("Invalid resource staging recovery journal")
+    parent = destination.parent.resolve()
+    transaction = state["transaction"]
+    return (_checked_path(parent / f".resources.stage-{transaction}", parent),
+            _checked_path(parent / f".resources.backup-{transaction}", parent))
+
+
+def recover_staging(destination):
+    """Recover interrupted copy/swap while caller holds the staging lock.
+
+    This handles process interruption, not certified power-loss durability.
+    An unsuccessful restoration retains the journal and original backup.
+    """
+    parent = destination.parent.resolve()
+    destination = _checked_path(destination, parent)
+    journal = _checked_path(parent / ".searchi-stage.json", parent)
+    if not journal.exists():
+        return
+    if journal.stat().st_size > 4096:
+        raise RuntimeError("Oversized resource staging recovery journal")
+    state = json.loads(journal.read_text(encoding="utf-8"))
+    candidate, backup = _journal_paths(destination, state)
+    for directory in [destination, candidate, backup]:
+        if directory.exists():
+            _check_tree(directory)
+    marker = ".searchi-stage-complete"
+
+    def complete(directory):
+        path = directory / marker
+        return path.is_file() and path.stat().st_size == 32 and path.read_text() == state["transaction"]
+
+    if not destination.exists():
+        if backup.exists():
+            # If restoration fails, do not discard either recoverable directory.
+            _move_directory(backup, destination, parent)
+        elif candidate.exists() and complete(candidate):
+            _move_directory(candidate, destination, parent)
+        elif state["had_destination"]:
+            raise RuntimeError("Original staging directory and backup are both missing; recovery retained")
+    if destination.exists() and backup.exists():
+        if not complete(destination):
+            raise RuntimeError("Ambiguous resource staging recovery; original backup retained")
+        _remove_tree(backup, parent)
+    _remove_tree(candidate, parent)
+    if destination.exists():
+        completion = destination / marker
+        if completion.exists():
+            if not complete(destination):
+                raise RuntimeError("Invalid resource staging completion marker")
+            completion.unlink()
+    journal.unlink()
+
+
+def _previous_owned(destination):
+    names = {"searchi-gemma-worker.exe", "searchi-gemma-worker", "runtime-libraries.json",
+             "LLAMA-LICENSE.txt", "VULKAN-LICENSE.txt", "MSVC-REDISTRIBUTION.txt", "vulkan-1.dll"}
+    manifest_path = Path(__file__).with_name("model-manifest.json")
+    if manifest_path.stat().st_size > 65536:
+        raise RuntimeError("Pinned model manifest is oversized")
+    model_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    files = model_manifest.get("files") if isinstance(model_manifest, dict) else None
+    if not isinstance(files, list) or not 1 <= len(files) <= 4:
+        raise RuntimeError("Invalid pinned model manifest")
+    for record in files:
+        name = record.get("path") if isinstance(record, dict) else None
+        if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.gguf", name):
+            raise RuntimeError("Invalid pinned model resource path")
+        names.add(f"embeddinggemma2/{name}")
+    inventory = destination / "runtime-libraries.json"
+    if inventory.exists():
+        if inventory.stat().st_size > 65536:
+            raise RuntimeError("Previous runtime inventory is oversized")
+        previous = json.loads(inventory.read_text(encoding="utf-8"))
+        records = previous.get("libraries") if isinstance(previous, dict) else None
+        if not isinstance(records, list) or len(records) > 64:
+            raise RuntimeError("Invalid previous runtime inventory")
+        for record in records:
+            name = record.get("path") if isinstance(record, dict) else None
+            if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+\.dll", name):
+                raise RuntimeError("Invalid previous runtime library path")
+            names.add(name)
+    return names
+
+
+def _resource_key(path):
+    value = Path(path).as_posix()
+    return value.casefold() if os.name == "nt" else value
+
+
+def _preserved_snapshot(destination, owned):
+    """Bounded content/tree snapshot detects edits to resources being retained."""
+    if not destination.exists():
+        return None
+    _check_tree(destination)
+    snapshot = {}
+    count = 0
+    for parent, directories, files in os.walk(destination, followlinks=False):
+        for name in [*directories, *files]:
+            count += 1
+            if count > 100000:
+                raise RuntimeError("Resource staging exceeds its snapshot file-count limit")
+            path = Path(parent) / name
+            key = _resource_key(path.relative_to(destination))
+            if key in owned:
+                if not path.is_file():
+                    raise RuntimeError("Owned runtime resource is not a file")
+                continue
+            if path.is_dir():
+                snapshot[key] = ("directory",)
+            else:
+                record = _record(path)
+                metadata = path.stat()
+                snapshot[key] = ("file", *record, metadata.st_mtime_ns, metadata.st_ino, metadata.st_dev)
+    return snapshot
+
+
+def stage_resources(destination, sources, validate=None):
+    """Replace a verified complete resource directory and preserve other assets."""
+    destination = Path(destination).absolute()
+    if destination.parent != destination.parent.resolve():
+        raise RuntimeError("Staging parent resolves through a link outside its expected location")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    parent = destination.parent.resolve()
+    destination = _checked_path(destination, parent)
+    with staging_lock(parent):
+        recover_staging(destination)
+        if destination.exists():
+            _check_tree(destination)
+            if (destination / ".searchi-stage-complete").exists():
+                raise RuntimeError("Reserved staging completion marker exists without its journal")
+        owned = _previous_owned(destination)
+        owned_keys = {_resource_key(path) for path in owned}
+        preserved = _preserved_snapshot(destination, owned_keys)
+        records = []
+        targets = set()
+        for source, relative in sources:
+            relative = Path(relative)
+            if relative.is_absolute() or relative.anchor or relative.drive or ".." in relative.parts or not relative.parts:
+                raise RuntimeError("Invalid staging resource target")
+            key = relative.as_posix().lower()
+            if key in targets:
+                raise RuntimeError("Duplicate staging resource target")
+            targets.add(key)
+            records.append((Path(source), relative, _record(source)))
+        state = {"schema": 1, "owner": "searchi-runtime-staging", "transaction": uuid.uuid4().hex,
+                 "had_destination": destination.exists()}
+        candidate, backup = _journal_paths(destination, state)
+        journal = _checked_path(parent / ".searchi-stage.json", parent)
+        pending = _checked_path(parent / f".searchi-stage-journal-{state['transaction']}.pending", parent)
+        try:
+            with pending.open("x", encoding="utf-8") as stream:
+                json.dump(state, stream)
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(pending, journal)
+        finally:
+            pending.unlink(missing_ok=True)
+        try:
+            if destination.exists():
+                def ignore_owned(directory, names):
+                    relative = Path(directory).relative_to(destination)
+                    return [name for name in names if _resource_key(relative / name) in owned_keys]
+
+                shutil.copytree(destination, candidate, copy_function=_verified_copy, ignore=ignore_owned)
+            else:
+                candidate.mkdir()
+            model_directory = candidate / "embeddinggemma2"
+            if model_directory.is_dir() and not any(model_directory.iterdir()):
+                if not model_directory.resolve().is_relative_to(candidate.resolve()):
+                    raise RuntimeError("Owned model directory escapes candidate")
+                model_directory.rmdir()
+            for source, relative, expected in records:
+                target = candidate / relative
+                if not target.resolve().is_relative_to(candidate.resolve()):
+                    raise RuntimeError("Staging resource target escapes candidate directory")
+                target.parent.mkdir(parents=True, exist_ok=True)
+                _verified_copy(source, target, expected)
+            _check_tree(candidate)
+            if validate:
+                validate(candidate)
+            if _preserved_snapshot(destination, owned_keys) != preserved:
+                raise RuntimeError("Preserved resources changed during staging; original retained")
+            (candidate / ".searchi-stage-complete").write_text(state["transaction"])
+            if destination.exists():
+                _move_directory(destination, backup, parent)
+            _move_directory(candidate, destination, parent)
+            recover_staging(destination)
+        except Exception:
+            recover_staging(destination)
+            raise
+
+
 def verify_runtime(source, required=False):
     """Preflight an explicit, bounded dependency inventory, never a DLL glob."""
     inventory = source.parent / "runtime-libraries.json"
@@ -74,6 +385,12 @@ def main():
     parser.add_argument("--backend", choices=["cpu", "auto", "metal", "vulkan", "cuda"], default="auto")
     parser.add_argument("--bundle-model", action="store_true")
     args = parser.parse_args()
+    destination = ROOT / "apps/desktop/src-tauri/resources"
+    if destination.parent.absolute() != destination.parent.resolve():
+        raise RuntimeError("Staging parent resolves through a link outside its expected location")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with staging_lock(destination.parent):
+        recover_staging(destination)
     name = "searchi-gemma-worker.exe" if os.name == "nt" else "searchi-gemma-worker"
     source = ROOT / ".searchi/runtime" / args.backend
     if os.name == "nt":
@@ -91,29 +408,30 @@ def main():
             raise RuntimeError("Microsoft CRT redistribution notice is missing")
         dependencies.append(crt_notice)
     artifacts = []
+    model_manifest = None
     if args.bundle_model:
-        manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
-        artifacts = verify_bundle(ROOT, manifest)
+        model_manifest = json.loads(Path(__file__).with_name("model-manifest.json").read_text())
+        artifacts = verify_bundle(ROOT, model_manifest)
     loader = None
     if os.name == "nt" and args.backend in {"auto", "vulkan"}:
         from provision_vulkan import provision
         _, loader = provision()
-    destination = ROOT / "apps/desktop/src-tauri/resources"
-    destination.mkdir(parents=True, exist_ok=True)
-    shutil.copy2(source, destination / name)
-    for dependency in dependencies:
-        shutil.copy2(dependency, destination / dependency.name)
-    # Include the upstream runtime license with the statically linked helper.
-    shutil.copy2(license_path, destination / "LLAMA-LICENSE.txt")
+    sources = [(source, source.name), (license_path, "LLAMA-LICENSE.txt")]
+    sources.extend((path, path.name) for path in dependencies)
     if loader:
         for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
             if name not in {path.name for path in dependencies}:
-                shutil.copy2(loader / name, destination / name)
-    if args.bundle_model:
-        model_destination = destination / "embeddinggemma2"
-        model_destination.mkdir(exist_ok=True)
-        for artifact in artifacts:
-            shutil.copy2(artifact, model_destination / artifact.name)
+                sources.append((loader / name, name))
+    sources.extend((path, f"embeddinggemma2/{path.name}") for path in artifacts)
+
+    def validate(candidate):
+        verify_runtime(candidate / source.name, os.name == "nt" and args.backend in {"auto", "vulkan"})
+        if model_manifest:
+            for artifact, spec in zip(artifacts, model_manifest["files"], strict=True):
+                if _record(candidate / "embeddinggemma2" / artifact.name) != (spec["bytes"], spec["sha256"]):
+                    raise RuntimeError("Copied model checksum mismatch")
+
+    stage_resources(destination, sources, validate)
     print(f"Staged native worker: {destination / source.name}")
 
 

tools/embeddinggemma2/test_stage_runtime.py

@@ -2,6 +2,7 @@
 import hashlib
 import json
 import os
+import subprocess
 from pathlib import Path
 import tempfile
 import unittest
@@ -11,6 +12,371 @@ import stage_runtime
 
 
 class StagingTests(unittest.TestCase):
+    def resources(self, directory):
+        root = Path(directory)
+        destination = root / "src-tauri/resources"
+        destination.mkdir(parents=True)
+        (destination / "searchi-gemma-worker.exe").write_bytes(b"previous worker")
+        (destination / "fonts").mkdir()
+        (destination / "fonts/Inter.woff2").write_bytes(b"font")
+        (destination / "media-runtime").mkdir()
+        (destination / "media-runtime/ffmpeg.exe").write_bytes(b"media")
+        (destination / "user-note.txt").write_bytes(b"preserve user file")
+        source = root / "worker.exe"
+        source.write_bytes(b"next worker")
+        return destination, source
+
+    def test_failed_copy_preserves_previous_complete_resources(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.shutil.copy2
+
+            def fail_worker(path, target, *args, **kwargs):
+                if Path(path) == source:
+                    raise OSError("injected copy failure")
+                return original(path, target, *args, **kwargs)
+
+            with patch.object(stage_runtime.shutil, "copy2", fail_worker):
+                with self.assertRaisesRegex(OSError, "copy failure"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            self.assertEqual((destination / "user-note.txt").read_bytes(), b"preserve user file")
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+            self.assertEqual(list(destination.parent.glob(".resources.stage-*")), [])
+
+    def test_changed_copy_bytes_reject_before_swap(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.shutil.copy2
+
+            def corrupt_worker(path, target, *args, **kwargs):
+                result = original(path, target, *args, **kwargs)
+                if Path(path) == source:
+                    Path(target).write_bytes(b"bad worker!")
+                return result
+
+            with patch.object(stage_runtime.shutil, "copy2", corrupt_worker):
+                with self.assertRaisesRegex(RuntimeError, "changed or copy was incomplete"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+
+    def test_source_change_after_preflight_rejects_before_swap(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.shutil.copy2
+
+            def mutate_source(path, target, *args, **kwargs):
+                if Path(path) == source:
+                    source.write_bytes(b"evil worker")
+                return original(path, target, *args, **kwargs)
+
+            with patch.object(stage_runtime.shutil, "copy2", mutate_source):
+                with self.assertRaisesRegex(RuntimeError, "changed or copy was incomplete"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+
+    def test_resource_targets_reject_traversal_and_windows_drive_roots(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            targets = ["../outside"]
+            if os.name == "nt":
+                targets.extend(["C:outside", "\\outside"])
+            for target in targets:
+                with self.subTest(target=target), self.assertRaisesRegex(RuntimeError, "resource target"):
+                    stage_runtime.stage_resources(destination, [(source, target)])
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+
+    def test_locked_destination_rename_preserves_old_resources(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.os.replace
+
+            def fail_old_rename(path, target):
+                if Path(path) == destination:
+                    raise PermissionError("destination is locked")
+                return original(path, target)
+
+            with patch.object(stage_runtime.os, "replace", fail_old_rename):
+                with self.assertRaisesRegex(PermissionError, "locked"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+
+    def test_interrupted_between_renames_restores_old_before_source_preflight(self):
+        class Interrupt(BaseException):
+            pass
+
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.os.replace
+
+            def interrupt_candidate(path, target):
+                if Path(path).name.startswith(".resources.stage-"):
+                    raise Interrupt()
+                return original(path, target)
+
+            with patch.object(stage_runtime.os, "replace", interrupt_candidate):
+                with self.assertRaises(Interrupt):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertFalse(destination.exists())
+            self.assertEqual(len(list(destination.parent.glob(".resources.backup-*"))), 1)
+            with self.assertRaisesRegex(RuntimeError, "source is missing"):
+                stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+
+    def test_interrupted_after_install_commits_verified_new_resources(self):
+        class Interrupt(BaseException):
+            pass
+
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.os.replace
+
+            def interrupt_after_candidate(path, target):
+                original(path, target)
+                if Path(path).name.startswith(".resources.stage-"):
+                    raise Interrupt()
+
+            with patch.object(stage_runtime.os, "replace", interrupt_after_candidate):
+                with self.assertRaises(Interrupt):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            with self.assertRaisesRegex(RuntimeError, "source is missing"):
+                stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"next worker")
+            self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
+            self.assertEqual((destination / "media-runtime/ffmpeg.exe").read_bytes(), b"media")
+            self.assertEqual((destination / "user-note.txt").read_bytes(), b"preserve user file")
+            self.assertEqual(list(destination.parent.glob(".resources.backup-*")), [])
+            self.assertFalse((destination / ".searchi-stage-complete").exists())
+
+    def test_failed_rollback_retains_backup_and_journal_for_retry(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.os.replace
+
+            def fail_candidate_and_restore(path, target):
+                if Path(path).name.startswith((".resources.stage-", ".resources.backup-")):
+                    raise PermissionError("injected rename failure")
+                return original(path, target)
+
+            with patch.object(stage_runtime.os, "replace", fail_candidate_and_restore):
+                with self.assertRaisesRegex(PermissionError, "rename failure"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            backup = next(destination.parent.glob(".resources.backup-*"))
+            self.assertEqual((backup / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            self.assertTrue((destination.parent / ".searchi-stage.json").exists())
+            with self.assertRaisesRegex(RuntimeError, "source is missing"):
+                stage_runtime.stage_resources(destination, [(source.with_name("missing"), "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+
+    def test_profile_switch_removes_only_previous_inventory_libraries(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            old = destination / "ggml-cpu-old.dll"
+            old.write_bytes(b"old backend")
+            (destination / "user-plugin.dll").write_bytes(b"user plugin")
+            (destination / "runtime-libraries.json").write_text(json.dumps({"libraries": [{"path": old.name}]}))
+            stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertFalse((destination / old.name).exists())
+            self.assertFalse((destination / "runtime-libraries.json").exists())
+            self.assertEqual((destination / "user-plugin.dll").read_bytes(), b"user plugin")
+            self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
+
+    def test_offline_to_online_removes_only_exact_pinned_model_pair(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            model_directory = destination / "embeddinggemma2"
+            model_directory.mkdir()
+            manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
+            for spec in manifest["files"]:
+                (model_directory / spec["path"]).write_bytes(b"old unverified model")
+            (model_directory / "user-note.txt").write_bytes(b"preserve unrelated asset")
+            stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            for spec in manifest["files"]:
+                self.assertFalse((destination / "embeddinggemma2" / spec["path"]).exists())
+            self.assertEqual((destination / "embeddinggemma2/user-note.txt").read_bytes(), b"preserve unrelated asset")
+            self.assertEqual((destination / "fonts/Inter.woff2").read_bytes(), b"font")
+
+    def test_online_staging_removes_empty_owned_model_directory(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            model_directory = destination / "embeddinggemma2"
+            model_directory.mkdir()
+            manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
+            for spec in manifest["files"]:
+                (model_directory / spec["path"]).write_bytes(b"old model")
+            stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertFalse((destination / "embeddinggemma2").exists())
+
+    def test_old_owned_worker_and_model_pair_are_never_copied(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            model_directory = destination / "embeddinggemma2"
+            model_directory.mkdir()
+            manifest = json.loads(Path(stage_runtime.__file__).with_name("model-manifest.json").read_text())
+            old_paths = {destination / "searchi-gemma-worker.exe"}
+            for spec in manifest["files"]:
+                path = model_directory / spec["path"]
+                path.write_bytes(b"old owned model")
+                old_paths.add(path)
+            original = stage_runtime.shutil.copy2
+
+            original_record = stage_runtime._record
+
+            def reject_owned_copy(path, target, *args, **kwargs):
+                if Path(path) in old_paths:
+                    self.fail("Previously owned runtime/model asset was unnecessarily copied")
+                return original(path, target, *args, **kwargs)
+
+            def reject_owned_hash(path):
+                if Path(path) in old_paths:
+                    self.fail("Previously owned runtime/model asset was unnecessarily hashed")
+                return original_record(path)
+
+            with patch.object(stage_runtime.shutil, "copy2", reject_owned_copy), patch.object(stage_runtime, "_record", reject_owned_hash):
+                stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"next worker")
+            self.assertFalse((destination / "embeddinggemma2").exists())
+
+    def test_concurrent_preserved_edit_aborts_and_keeps_latest_original(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+
+            def edit_original(_candidate):
+                (destination / "user-note.txt").write_bytes(b"latest user edit")
+
+            with self.assertRaisesRegex(RuntimeError, "Preserved resources changed"):
+                stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")], edit_original)
+            self.assertEqual((destination / "user-note.txt").read_bytes(), b"latest user edit")
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+
+    def test_concurrent_preserved_addition_aborts_and_keeps_new_file(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+
+            def add_original(_candidate):
+                (destination / "fonts/new-font.woff2").write_bytes(b"new user asset")
+
+            with self.assertRaisesRegex(RuntimeError, "Preserved resources changed"):
+                stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")], add_original)
+            self.assertEqual((destination / "fonts/new-font.woff2").read_bytes(), b"new user asset")
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+
+    def test_concurrent_staging_lock_fails_without_mutation(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            with stage_runtime.staging_lock(destination.parent):
+                with self.assertRaisesRegex(RuntimeError, "holds the lock"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+
+    def test_malicious_recovery_journal_cannot_escape_sibling_paths(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            outside = Path(directory) / "outside"
+            outside.mkdir()
+            (outside / "user-data.txt").write_bytes(b"never remove")
+            journal = destination.parent / ".searchi-stage.json"
+            journal.write_text(json.dumps({"schema": 1, "owner": "searchi-runtime-staging", "transaction": "../outside", "had_destination": True}))
+            with self.assertRaisesRegex(RuntimeError, "recovery journal"):
+                stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((outside / "user-data.txt").read_bytes(), b"never remove")
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+
+    def test_existing_unrelated_symlink_rejects_before_stage_changes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            outside = Path(directory) / "outside.txt"
+            outside.write_bytes(b"private data")
+            link = destination / "user-link.txt"
+            try:
+                link.symlink_to(outside)
+            except OSError:
+                self.skipTest("Host does not permit creating test symlinks")
+            try:
+                with self.assertRaisesRegex(RuntimeError, "symlinks or junctions"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+                self.assertEqual(outside.read_bytes(), b"private data")
+                self.assertTrue(link.is_symlink())
+                self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            finally:
+                link.unlink()
+
+    def test_first_install_interrupt_after_verified_candidate_recovers(self):
+        class Interrupt(BaseException):
+            pass
+
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            destination = root / "src-tauri/resources"
+            source = root / "worker.exe"
+            source.write_bytes(b"first worker")
+            original = stage_runtime.os.replace
+
+            def interrupt_candidate(path, target):
+                if Path(path).name.startswith(".resources.stage-"):
+                    raise Interrupt()
+                return original(path, target)
+
+            with patch.object(stage_runtime.os, "replace", interrupt_candidate):
+                with self.assertRaises(Interrupt):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertFalse(destination.exists())
+            with stage_runtime.staging_lock(destination.parent):
+                stage_runtime.recover_staging(destination)
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"first worker")
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+
+    @unittest.skipUnless(os.name == "nt", "Windows junction boundary regression")
+    def test_existing_junction_rejects_without_following_outside_directory(self):
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            outside = Path(directory) / "outside"
+            outside.mkdir()
+            (outside / "private.txt").write_bytes(b"untouched")
+            link = destination / "user-junction"
+            result = subprocess.run(["cmd.exe", "/c", "mklink", "/J", str(link), str(outside)], capture_output=True)
+            if result.returncode:
+                self.skipTest("Host does not permit creating a test junction")
+            try:
+                self.assertTrue(stage_runtime._is_link(link))
+                with self.assertRaisesRegex(RuntimeError, "symlinks or junctions"):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+                self.assertEqual((outside / "private.txt").read_bytes(), b"untouched")
+                self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            finally:
+                # Remove only the verified test junction, never its target tree.
+                self.assertEqual(link.absolute().parent, destination.absolute())
+                self.assertTrue(stage_runtime._is_link(link))
+                link.rmdir()
+
+    def test_interrupted_partial_candidate_keeps_old_resources_and_recovers(self):
+        class Interrupt(BaseException):
+            pass
+
+        with tempfile.TemporaryDirectory() as directory:
+            destination, source = self.resources(directory)
+            original = stage_runtime.shutil.copy2
+
+            def interrupt_copy(path, target, *args, **kwargs):
+                if Path(path) == source:
+                    Path(target).write_bytes(b"partial")
+                    raise Interrupt()
+                return original(path, target, *args, **kwargs)
+
+            with patch.object(stage_runtime.shutil, "copy2", interrupt_copy):
+                with self.assertRaises(Interrupt):
+                    stage_runtime.stage_resources(destination, [(source, "searchi-gemma-worker.exe")])
+            self.assertEqual((destination / "searchi-gemma-worker.exe").read_bytes(), b"previous worker")
+            with stage_runtime.staging_lock(destination.parent):
+                stage_runtime.recover_staging(destination)
+            self.assertEqual(list(destination.parent.glob(".resources.stage-*")), [])
+            self.assertFalse((destination.parent / ".searchi-stage.json").exists())
+
     @unittest.skipUnless(os.name == "nt", "Windows app-local CRT staging")
     def test_missing_crt_notice_preserves_existing_staged_worker(self):
         with tempfile.TemporaryDirectory() as directory: