karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Keep prepared search readers warm and verify abrupt indexing recovery

Browse Source
This commit is contained in:
karl committed 2026-10-13 01:36:34 +02:00
1 parent 423a1ea57d
commit 6b78c8bb7f
17 files changed
+1011 -41
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/package_windows.py
+76 -5
View File
@@ -10,11 +10,80 @@ import os
import re
from pathlib import Path
import struct
import stat
import tempfile
import zipfile
ROOT = Path(__file__).resolve().parents[1]
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
MAX_MEMBER = 256 * 1024 * 1024
MAX_MEDIA_TOTAL = 1024 * 1024 * 1024
def digest(path):
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def media_provenance(root, resources, architecture):
"""Compare staged helpers/notices with the checksum-pinned upstream ZIP."""
manifest_path = root / "tools/media-runtime/manifest.json"
if not manifest_path.is_file() or manifest_path.stat().st_size > 65536:
raise ValueError("Required pinned media manifest is missing or oversized")
manifest_bytes = manifest_path.read_bytes()
manifest = json.loads(manifest_bytes)
artifact = manifest["platforms"][f"windows-{architecture}"]
name = artifact.get("archive")
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+\.zip", name):
raise ValueError("Invalid media archive name")
archive = root / ".searchi/media-runtime" / name
if (not archive.is_file() or archive.is_symlink() or archive.absolute() != archive.resolve()
or archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]):
raise ValueError("Media archive provenance mismatch")
staged_manifest = resources / "media-runtime/manifest.json"
if (not staged_manifest.is_file() or staged_manifest.stat().st_size != len(manifest_bytes)
or staged_manifest.read_bytes() != manifest_bytes):
raise ValueError("Staged media manifest differs from pinned manifest")
selected = {"media-runtime/manifest.json": {"bytes": len(manifest_bytes), "sha256": hashlib.sha256(manifest_bytes).hexdigest()}}
selected_keys = {name.casefold() for name in selected}
with zipfile.ZipFile(archive) as package:
members = package.infolist()
if len(members) > 10000:
raise ValueError("Media archive exceeds member ceiling")
total = 0
for member in members:
total += member.file_size
if member.file_size < 0 or member.file_size > MAX_MEMBER or total > MAX_MEDIA_TOTAL:
raise ValueError("Media archive exceeds size ceiling")
parts = member.filename.replace("\\", "/").split("/")
basename = parts[-1]
if basename in {"ffmpeg.exe", "ffprobe.exe"}:
target = "media-runtime/bin/" + basename
elif basename.lower().startswith(("license", "copying")) or basename.lower() in {"readme.txt", "readme.md"}:
if any(part in {"", ".", ".."} or ":" in part for part in parts):
raise ValueError("Unsafe media notice path")
target = "media-runtime/licenses/" + "/".join(parts)
else:
continue
mode = member.external_attr >> 16
if (any(part in {"", ".", ".."} or ":" in part for part in parts)
or member.is_dir() or stat.S_IFMT(mode) not in {0, stat.S_IFREG} or member.flag_bits & 1
or member.file_size <= 0 or target.casefold() in selected_keys):
raise ValueError("Invalid or duplicate media member")
path = resources / target
if (not path.is_file() or path.is_symlink() or path.absolute() != path.resolve()
or not path.resolve().is_relative_to(resources.resolve())):
raise ValueError(f"Staged media helper/notice missing or outside resources: {target}")
with package.open(member) as stream:
checksum = hashlib.file_digest(stream, "sha256").hexdigest()
if path.stat().st_size != member.file_size or digest(path) != checksum:
raise ValueError(f"Media helper/notice differs from pinned archive: {target}")
selected[target] = {"bytes": member.file_size, "sha256": checksum}
selected_keys.add(target.casefold())
if (not {"media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe"}.issubset(selected)
or not any(name.startswith("media-runtime/licenses/") for name in selected)):
raise ValueError("Media archive lacks required helpers or notices")
return selected
def pe_machine(path):
@@ -34,6 +103,10 @@ def pe_machine(path):
def package(root, executable, output):
resources = root / "apps/desktop/src-tauri/resources"
machine = pe_machine(executable)
architecture = {0x8664: "x86_64", 0xAA64: "aarch64"}.get(machine)
if architecture is None:
raise ValueError("Unsupported Windows architecture")
inventory = resources / "runtime-libraries.json"
if not inventory.is_file() or inventory.stat().st_size > 65536:
raise ValueError("Required runtime dependency inventory is missing or oversized")
@@ -69,6 +142,8 @@ def package(root, executable, output):
raise ValueError(f"Runtime checksum mismatch: {name}")
if "bytes" in record and path.stat().st_size != record["bytes"]:
raise ValueError(f"Runtime size mismatch: {name}")
media = media_provenance(root, resources, architecture)
expected.update(media)
selected = [(executable, "searchi.exe"), (root / "LICENSE", "LICENSE.txt")]
for name in ["searchi-gemma-worker.exe", "LLAMA-LICENSE.txt", "MSVC-REDISTRIBUTION.txt", "vulkan-1.dll", "VULKAN-LICENSE.txt", "media-runtime/manifest.json",
"media-runtime/bin/ffmpeg.exe", "media-runtime/bin/ffprobe.exe",
@@ -77,17 +152,13 @@ def package(root, executable, output):
selected.append((inventory, inventory.name))
selected_names = {name for _, name in selected}
selected.extend((resources / name, name) for name in expected if name not in selected_names)
for directory in ["licenses", "media-runtime/licenses"]:
for directory in ["licenses"]:
notices = list((resources / directory).rglob("*.txt"))
if not notices:
raise ValueError(f"Required notices are missing: {directory}")
selected_names = {name for _, name in selected}
selected.extend((path, path.relative_to(resources).as_posix()) for path in notices
if path.relative_to(resources).as_posix() not in selected_names)
machine = pe_machine(executable)
architecture = {0x8664: "x86_64", 0xAA64: "aarch64"}.get(machine)
if architecture is None:
raise ValueError("Unsupported Windows architecture")
for path, name in selected:
if not path.is_file():
raise ValueError(f"Required package file is missing: {name}")
tools/test_package_windows.py
+71
View File
@@ -45,9 +45,80 @@ class PackageTests(unittest.TestCase):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("notice")
(resources / "settings.json").write_text("private settings")
media = root / ".searchi/media-runtime"
media.mkdir(parents=True)
upstream = media / "pinned-windows.zip"
with zipfile.ZipFile(upstream, "w") as archive:
for staged, member in [("bin/ffmpeg.exe", "bin/ffmpeg.exe"), ("bin/ffprobe.exe", "bin/ffprobe.exe"), ("licenses/LICENSE.txt", "LICENSE.txt")]:
archive.writestr(member, (resources / "media-runtime" / staged).read_bytes())
manifest = root / "tools/media-runtime/manifest.json"
manifest.parent.mkdir(parents=True)
manifest.write_text(json.dumps({"platforms": {"windows-x86_64": {
"archive": upstream.name, "bytes": upstream.stat().st_size,
"sha256": hashlib.sha256(upstream.read_bytes()).hexdigest()}}}))
(resources / "media-runtime/manifest.json").write_bytes(manifest.read_bytes())
inventory(resources)
return root, resources, app, root / "archive.zip"
def test_valid_pe_media_mutation_and_notice_corruption_preserve_previous_archive(self):
for name in ["bin/ffmpeg.exe", "bin/ffprobe.exe", "licenses/LICENSE.txt", "manifest.json"]:
with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
root, resources, app, output = self.fixture(directory)
output.write_bytes(b"previous package")
path = resources / "media-runtime" / name
path.write_bytes(path.read_bytes() + b"tampered")
with self.assertRaisesRegex(ValueError, "differs from pinned"):
package(root, app, output)
self.assertEqual(output.read_bytes(), b"previous package")
self.assertEqual(list(output.parent.glob("*.partial")), [])
def test_pinned_archive_corruption_preserves_previous_package(self):
with tempfile.TemporaryDirectory() as directory:
root, _, app, output = self.fixture(directory)
output.write_bytes(b"previous package")
source = root / ".searchi/media-runtime/pinned-windows.zip"
source.write_bytes(source.read_bytes() + b"changed")
with self.assertRaisesRegex(ValueError, "provenance mismatch"):
package(root, app, output)
self.assertEqual(output.read_bytes(), b"previous package")
def test_media_change_during_archive_write_preserves_previous_package(self):
with tempfile.TemporaryDirectory() as directory:
root, resources, app, output = self.fixture(directory)
output.write_bytes(b"previous package")
original = zipfile.ZipFile.open
def mutate(archive, name, mode="r", *args, **kwargs):
if name == "media-runtime/bin/ffmpeg.exe" and mode == "w":
path = resources / name
path.write_bytes(path.read_bytes() + b"changed")
return original(archive, name, mode, *args, **kwargs)
with mock.patch.object(zipfile.ZipFile, "open", mutate), self.assertRaisesRegex(ValueError, "changed while packaging"):
package(root, app, output)
self.assertEqual(output.read_bytes(), b"previous package")
def test_duplicate_helper_and_link_notice_are_rejected_before_output(self):
for kind in ["duplicate", "link"]:
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory:
root, resources, app, output = self.fixture(directory)
output.write_bytes(b"previous package")
upstream = root / ".searchi/media-runtime/pinned-windows.zip"
with zipfile.ZipFile(upstream, "a") as archive:
if kind == "duplicate":
archive.writestr("other/ffmpeg.exe", (resources / "media-runtime/bin/ffmpeg.exe").read_bytes())
else:
entry = zipfile.ZipInfo("COPYING.txt")
entry.create_system = 3
entry.external_attr = 0o120777 << 16
archive.writestr(entry, "somewhere")
manifest = root / "tools/media-runtime/manifest.json"
data = json.loads(manifest.read_text())
data["platforms"]["windows-x86_64"].update(bytes=upstream.stat().st_size, sha256=hashlib.sha256(upstream.read_bytes()).hexdigest())
manifest.write_text(json.dumps(data))
(resources / "media-runtime/manifest.json").write_bytes(manifest.read_bytes())
with self.assertRaisesRegex(ValueError, "Invalid or duplicate"):
package(root, app, output)
self.assertEqual(output.read_bytes(), b"previous package")
def test_complete_archive_excludes_settings_and_verifies_recorded_bytes(self):
with tempfile.TemporaryDirectory() as directory:
root, _, app, output = self.fixture(directory)