Keep prepared search readers warm and verify abrupt indexing recovery

tools/package_windows.py

@@ -10,11 +10,80 @@ import os
 import re
 from pathlib import Path
 import struct
+import stat
 import tempfile
 import zipfile
 
 ROOT = Path(__file__).resolve().parents[1]
 CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
+MAX_MEMBER = 256 * 1024 * 1024
+MAX_MEDIA_TOTAL = 1024 * 1024 * 1024
+
+
+def digest(path):
+    with path.open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def media_provenance(root, resources, architecture):
+    """Compare staged helpers/notices with the checksum-pinned upstream ZIP."""
+    manifest_path = root / "tools/media-runtime/manifest.json"
+    if not manifest_path.is_file() or manifest_path.stat().st_size > 65536:
+        raise ValueError("Required pinned media manifest is missing or oversized")
+    manifest_bytes = manifest_path.read_bytes()
+    manifest = json.loads(manifest_bytes)
+    artifact = manifest["platforms"][f"windows-{architecture}"]
+    name = artifact.get("archive")
+    if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+\.zip", name):
+        raise ValueError("Invalid media archive name")
+    archive = root / ".searchi/media-runtime" / name
+    if (not archive.is_file() or archive.is_symlink() or archive.absolute() != archive.resolve()
+            or archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]):
+        raise ValueError("Media archive provenance mismatch")
+    staged_manifest = resources / "media-runtime/manifest.json"
+    if (not staged_manifest.is_file() or staged_manifest.stat().st_size != len(manifest_bytes)
+            or staged_manifest.read_bytes() != manifest_bytes):
+        raise ValueError("Staged media manifest differs from pinned manifest")
+    selected = {"media-runtime/manifest.json": {"bytes": len(manifest_bytes), "sha256": hashlib.sha256(manifest_bytes).hexdigest()}}
+    selected_keys = {name.casefold() for name in selected}
+    with zipfile.ZipFile(archive) as package:
+        members = package.infolist()
+        if len(members) > 10000:
+            raise ValueError("Media archive exceeds member ceiling")
+        total = 0
+        for member in members:
+            total += member.file_size
+            if member.file_size < 0 or member.file_size > MAX_MEMBER or total > MAX_MEDIA_TOTAL:
+                raise ValueError("Media archive exceeds size ceiling")
+            parts = member.filename.replace("\\", "/").split("/")
+            basename = parts[-1]
+            if basename in {"ffmpeg.exe", "ffprobe.exe"}:
+                target = "media-runtime/bin/" + basename
+            elif basename.lower().startswith(("license", "copying")) or basename.lower() in {"readme.txt", "readme.md"}:
+                if any(part in {"", ".", ".."} or ":" in part for part in parts):
+                    raise ValueError("Unsafe media notice path")
+                target = "media-runtime/licenses/" + "/".join(parts)
+            else:
+                continue
+            mode = member.external_attr >> 16
+            if (any(part in {"", ".", ".."} or ":" in part for part in parts)
+                    or member.is_dir() or stat.S_IFMT(mode) not in {0, stat.S_IFREG} or member.flag_bits & 1
+                    or member.file_size <= 0 or target.casefold() in selected_keys):
+                raise ValueError("Invalid or duplicate media member")
+            path = resources / target
+            if (not path.is_file() or path.is_symlink() or path.absolute() != path.resolve()
+                    or not path.resolve().is_relative_to(resources.resolve())):
+                raise ValueError(f"Staged media helper/notice missing or outside resources: {target}")
+            with package.open(member) as stream:
+                checksum = hashlib.file_digest(stream, "sha256").hexdigest()
+            if path.stat().st_size != member.file_size or digest(path) != checksum:
+                raise ValueError(f"Media helper/notice differs from pinned archive: {target}")
+            selected[target] = {"bytes": member.file_size, "sha256": checksum}
+            selected_keys.add(target.casefold())
+    if (not {"media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe"}.issubset(selected)
+            or not any(name.startswith("media-runtime/licenses/") for name in selected)):
+        raise ValueError("Media archive lacks required helpers or notices")
+    return selected
 
 
 def pe_machine(path):
@@ -34,6 +103,10 @@ def pe_machine(path):
 
 def package(root, executable, output):
     resources = root / "apps/desktop/src-tauri/resources"
+    machine = pe_machine(executable)
+    architecture = {0x8664: "x86_64", 0xAA64: "aarch64"}.get(machine)
+    if architecture is None:
+        raise ValueError("Unsupported Windows architecture")
     inventory = resources / "runtime-libraries.json"
     if not inventory.is_file() or inventory.stat().st_size > 65536:
         raise ValueError("Required runtime dependency inventory is missing or oversized")
@@ -69,6 +142,8 @@ def package(root, executable, output):
                 raise ValueError(f"Runtime checksum mismatch: {name}")
         if "bytes" in record and path.stat().st_size != record["bytes"]:
             raise ValueError(f"Runtime size mismatch: {name}")
+    media = media_provenance(root, resources, architecture)
+    expected.update(media)
     selected = [(executable, "searchi.exe"), (root / "LICENSE", "LICENSE.txt")]
     for name in ["searchi-gemma-worker.exe", "LLAMA-LICENSE.txt", "MSVC-REDISTRIBUTION.txt", "vulkan-1.dll", "VULKAN-LICENSE.txt", "media-runtime/manifest.json",
                  "media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe",
@@ -77,17 +152,13 @@ def package(root, executable, output):
     selected.append((inventory, inventory.name))
     selected_names = {name for _, name in selected}
     selected.extend((resources / name, name) for name in expected if name not in selected_names)
-    for directory in ["licenses", "media-runtime/licenses"]:
+    for directory in ["licenses"]:
         notices = list((resources / directory).rglob("*.txt"))
         if not notices:
             raise ValueError(f"Required notices are missing: {directory}")
         selected_names = {name for _, name in selected}
         selected.extend((path, path.relative_to(resources).as_posix()) for path in notices
                         if path.relative_to(resources).as_posix() not in selected_names)
-    machine = pe_machine(executable)
-    architecture = {0x8664: "x86_64", 0xAA64: "aarch64"}.get(machine)
-    if architecture is None:
-        raise ValueError("Unsupported Windows architecture")
     for path, name in selected:
         if not path.is_file():
             raise ValueError(f"Required package file is missing: {name}")