Package portable Linux runtime and verify Google CPU search
This commit is contained in:
karl committed
1 parent
e3558a0104
commit
0b56ab553c
22 files changed
+1585
-16
No files matched your search
@@ -0,0 +1,259 @@
|
||||
"""Package an already-built Linux x86-64 headless development candidate.
|
||||
|
||||
No artifact is executed. readelf inspects dependency closure; native execution,
|
||||
ABI compatibility, GPU hardware and production installers remain release gates.
|
||||
"""
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_FILE = 256 * 1024 * 1024
|
||||
MAX_TOTAL = 1024 * 1024 * 1024
|
||||
CPU_VARIANTS = {"x64", "sse42", "sandybridge", "ivybridge", "piledriver", "haswell",
|
||||
"skylakex", "cannonlake", "cascadelake", "icelake", "cooperlake", "zen4",
|
||||
"alderlake", "sapphirerapids"}
|
||||
SYSTEM = {"libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0", "librt.so.1",
|
||||
"libmvec.so.1", "libstdc++.so.6", "libgcc_s.so.1", "ld-linux-x86-64.so.2"}
|
||||
|
||||
|
||||
def safe_name(name):
|
||||
if (not isinstance(name, str) or not name or "\\" in name or ":" in name
|
||||
or str(PurePosixPath(name)) != name or PurePosixPath(name).is_absolute()
|
||||
or any(part in {".", ".."} for part in name.split("/"))):
|
||||
raise ValueError("Unsafe package path")
|
||||
return name
|
||||
|
||||
|
||||
def regular(path, parent):
|
||||
path, parent = Path(path).absolute(), Path(parent).resolve()
|
||||
if not path.is_relative_to(parent) or not path.resolve().is_relative_to(parent):
|
||||
raise ValueError("Package source escapes its directory")
|
||||
candidate = path
|
||||
while candidate != parent:
|
||||
if candidate.is_symlink():
|
||||
raise ValueError("Package source contains a symlink")
|
||||
candidate = candidate.parent
|
||||
if not path.is_file() or not 0 < path.stat().st_size <= MAX_FILE:
|
||||
raise ValueError(f"Missing or oversized package source: {path.name}")
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def elf_x64(path):
|
||||
with path.open("rb") as stream:
|
||||
header = stream.read(64)
|
||||
if (len(header) != 64 or header[:7] != b"\x7fELF\x02\x01\x01"
|
||||
or struct.unpack_from("<H", header, 18)[0] != 62
|
||||
or struct.unpack_from("<H", header, 16)[0] not in {2, 3}):
|
||||
raise ValueError(f"Expected Linux x86-64 ELF artifact: {path.name}")
|
||||
|
||||
|
||||
def dynamic(path):
|
||||
output = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True, timeout=30)
|
||||
values = {}
|
||||
for key, value in re.findall(r"\((NEEDED|SONAME|RUNPATH|RPATH)\).*?\[([^\]]+)\]", output):
|
||||
values.setdefault(key, []).append(value)
|
||||
return values
|
||||
|
||||
|
||||
def runtime_files(runtime):
|
||||
inventory = runtime / "runtime-libraries.json"
|
||||
regular(inventory, runtime)
|
||||
if inventory.stat().st_size > 65536:
|
||||
raise ValueError("Oversized runtime inventory")
|
||||
inventory_bytes = inventory.read_bytes()
|
||||
manifest = json.loads(inventory_bytes)
|
||||
records = manifest.get("libraries") if isinstance(manifest, dict) else None
|
||||
if not isinstance(manifest, dict) or manifest.get("platform") != "linux" or not isinstance(records, list) or not 1 <= len(records) <= 64:
|
||||
raise ValueError("Invalid Linux runtime inventory")
|
||||
expected = {"searchi-gemma-worker": {"sha256": manifest.get("worker_sha256")}}
|
||||
core = {}
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
raise ValueError("Invalid library record")
|
||||
name = record.get("path")
|
||||
safe_name(name)
|
||||
match = re.fullmatch(r"lib(ggml|ggml-base|llama|mtmd)\.so(?:\.\d+)*", name)
|
||||
if match:
|
||||
if match[1] in core:
|
||||
raise ValueError("Duplicate runtime core library")
|
||||
core[match[1]] = name
|
||||
elif name != "libggml-vulkan.so" and name not in {f"libggml-cpu-{v}.so" for v in CPU_VARIANTS}:
|
||||
raise ValueError("Unapproved runtime library")
|
||||
if name in expected or type(record.get("bytes")) is not int or record["bytes"] <= 0:
|
||||
raise ValueError("Duplicate or invalid runtime library")
|
||||
expected[name] = record
|
||||
if set(core) != {"ggml", "ggml-base", "llama", "mtmd"} or not {"libggml-cpu-x64.so", "libggml-vulkan.so"}.issubset(expected):
|
||||
raise ValueError("Runtime lacks baseline CPU, Vulkan or core library")
|
||||
for name, record in expected.items():
|
||||
path = runtime / name
|
||||
regular(path, runtime)
|
||||
checksum = record.get("sha256")
|
||||
if not isinstance(checksum, str) or not re.fullmatch(r"[0-9a-f]{64}", checksum):
|
||||
raise ValueError("Invalid runtime checksum")
|
||||
if digest(path) != checksum or ("bytes" in record and path.stat().st_size != record["bytes"]):
|
||||
raise ValueError(f"Runtime integrity mismatch: {name}")
|
||||
record["bytes"] = path.stat().st_size
|
||||
elf_x64(path)
|
||||
metadata = dynamic(path)
|
||||
if metadata.get("RPATH") or metadata.get("RUNPATH") != ["$ORIGIN"]:
|
||||
raise ValueError("Runtime requires origin-only RUNPATH")
|
||||
allowed = set(expected) | SYSTEM
|
||||
if name == "libggml-vulkan.so":
|
||||
allowed.add("libvulkan.so.1")
|
||||
if set(metadata.get("NEEDED", [])) - allowed:
|
||||
raise ValueError(f"Unpackaged runtime dependency: {name}")
|
||||
if name in core.values() and metadata.get("SONAME") != [name]:
|
||||
raise ValueError("Core SONAME differs from packaged filename")
|
||||
expected[inventory.name] = {"sha256": hashlib.sha256(inventory_bytes).hexdigest(), "bytes": len(inventory_bytes)}
|
||||
return expected
|
||||
|
||||
|
||||
def media_files(root, media):
|
||||
manifest_path = root / "tools/media-runtime/manifest.json"
|
||||
regular(manifest_path, root)
|
||||
if manifest_path.stat().st_size > 65536:
|
||||
raise ValueError("Oversized media manifest")
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
artifact = manifest["platforms"]["linux-x86_64"]
|
||||
archive_name = safe_name(artifact["archive"])
|
||||
if "/" in archive_name or not archive_name.endswith(".tar.xz"):
|
||||
raise ValueError("Invalid media archive name")
|
||||
archive = media / archive_name
|
||||
regular(archive, media)
|
||||
if archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]:
|
||||
raise ValueError("Media archive provenance mismatch")
|
||||
selected = {}
|
||||
with tarfile.open(archive, "r:xz") as package:
|
||||
members = 0
|
||||
declared_total = 0
|
||||
for member in package:
|
||||
members += 1
|
||||
declared_total += member.size
|
||||
if (members > 10000 or member.size < 0 or member.size > MAX_FILE
|
||||
or declared_total > MAX_TOTAL):
|
||||
raise ValueError("Media archive exceeds member or size ceiling")
|
||||
basename = member.name.replace("\\", "/").split("/")[-1]
|
||||
if basename in {"ffmpeg", "ffprobe"}:
|
||||
target = f"bin/{basename}"
|
||||
elif basename.lower().startswith(("license", "copying")) or basename.lower() in {"readme.txt", "readme.md"}:
|
||||
target = "licenses/" + safe_name(member.name)
|
||||
else:
|
||||
continue
|
||||
if not member.isfile() or not 0 < member.size <= MAX_FILE or target in selected:
|
||||
raise ValueError("Invalid or duplicate media member")
|
||||
path = media / target
|
||||
regular(path, media)
|
||||
with package.extractfile(member) as stream:
|
||||
checksum = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
if path.stat().st_size != member.size or digest(path) != checksum:
|
||||
raise ValueError(f"Media helper/notice differs from pinned archive: {target}")
|
||||
selected[target] = {"sha256": checksum, "bytes": member.size}
|
||||
if not {"bin/ffmpeg", "bin/ffprobe"}.issubset(selected) or not any(n.startswith("licenses/") for n in selected):
|
||||
raise ValueError("Media archive lacks helpers or notices")
|
||||
selected = {"media-runtime/" + name: (media / name, record) for name, record in selected.items()}
|
||||
selected["media-runtime/manifest.json"] = (manifest_path, {"sha256": digest(manifest_path), "bytes": manifest_path.stat().st_size})
|
||||
return selected
|
||||
|
||||
|
||||
def package(root, executable, runtime, media, output, llama_license=None):
|
||||
root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output))
|
||||
regular(executable, executable.parent)
|
||||
elf_x64(executable)
|
||||
expected = runtime_files(runtime)
|
||||
cli_metadata = dynamic(executable)
|
||||
if (cli_metadata.get("RPATH") or cli_metadata.get("RUNPATH")
|
||||
or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
|
||||
raise ValueError("CLI has unexpected native dependency or search path")
|
||||
selected = {name: (runtime / name, record) for name, record in expected.items()}
|
||||
selected.update(media_files(root, media))
|
||||
selected["searchi"] = (executable, {"sha256": digest(executable), "bytes": executable.stat().st_size})
|
||||
license_path = Path(llama_license) if llama_license else root / ".searchi/references/llama.cpp/LICENSE"
|
||||
for path, name, parent in [(root / "LICENSE", "LICENSE.txt", root), (license_path, "LLAMA-LICENSE.txt", license_path.parent)]:
|
||||
regular(path, parent)
|
||||
selected[name] = (path, {"sha256": digest(path), "bytes": path.stat().st_size})
|
||||
for name in ("media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"):
|
||||
elf_x64(selected[name][0])
|
||||
metadata = dynamic(selected[name][0])
|
||||
if metadata.get("RPATH") or metadata.get("RUNPATH") or set(metadata.get("NEEDED", [])) - SYSTEM:
|
||||
raise ValueError("Media helper has unpackaged dependencies")
|
||||
if sum(record["bytes"] for _, record in selected.values()) > MAX_TOTAL:
|
||||
raise ValueError("Package exceeds total size ceiling")
|
||||
if output.resolve() in {path.resolve() for path, _ in selected.values()}:
|
||||
raise ValueError("Package output would overwrite an input")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=output.parent, suffix=".partial", delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
try:
|
||||
records = []
|
||||
with temporary_path.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive:
|
||||
for name, (path, expected_record) in sorted(selected.items()):
|
||||
safe_name(name)
|
||||
regular(path, path.parent)
|
||||
# Bounded immutable snapshot: archive and manifest describe the same bytes.
|
||||
with path.open("rb") as stream:
|
||||
data = stream.read(MAX_FILE + 1)
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]:
|
||||
raise ValueError(f"Source changed during packaging: {name}")
|
||||
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
|
||||
add(archive, name, data, mode)
|
||||
records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)})
|
||||
instructions = ("searchi — Find what matters. Locally.\n\n"
|
||||
"Extract the entire archive into its own folder. This is a headless development candidate, not a production installer.\n"
|
||||
"Use ./searchi --help. Setup downloads about 865 MB of verified Google model assets:\n"
|
||||
"./searchi setup --model-dir ./models\n"
|
||||
"./searchi index --db ./searchi.sqlite --root /path/to/files\n"
|
||||
"./searchi embed --db ./searchi.sqlite --model-dir ./models --embedding-worker ./searchi-gemma-worker\n"
|
||||
"./searchi search --db ./searchi.sqlite 'what you remember' --model-dir ./models --embedding-worker ./searchi-gemma-worker\n"
|
||||
"Media helpers are discovered automatically beside this CLI. Ordinary search stays local.\n"
|
||||
"The normal runtime includes Vulkan and CPU fallback. Graphics drivers and a compatible system Vulkan loader are optional.\n"
|
||||
"Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n").encode()
|
||||
add(archive, "START-HERE.txt", instructions, 0o644)
|
||||
records.append({"path": "START-HERE.txt", "bytes": len(instructions),
|
||||
"sha256": hashlib.sha256(instructions).hexdigest(), "mode": oct(0o644)})
|
||||
add(archive, "PACKAGE.json", json.dumps({"platform": "linux", "architecture": "x86_64",
|
||||
"status": "unsigned headless development candidate; distribution ABI and GPU hardware validation pending",
|
||||
"system_libraries": sorted(SYSTEM), "optional_gpu_loader": "libvulkan.so.1",
|
||||
"files": records}, indent=2).encode(), 0o644)
|
||||
os.replace(temporary_path, output)
|
||||
finally:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
return "x86_64"
|
||||
|
||||
|
||||
def add(archive, name, data, mode):
|
||||
info = tarfile.TarInfo(safe_name(name))
|
||||
info.size, info.mode, info.mtime = len(data), mode, 0
|
||||
info.uid = info.gid = 0
|
||||
info.uname = info.gname = ""
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--executable", type=Path, default=ROOT / "target/release/searchi")
|
||||
parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto")
|
||||
parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime")
|
||||
parser.add_argument("--llama-license", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=ROOT / ".searchi/packages/searchi-linux-x86_64.tar.gz")
|
||||
args = parser.parse_args()
|
||||
package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license)
|
||||
print(f"Packaged Linux x86_64 development candidate: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user