233 lines
12 KiB
Python
233 lines
12 KiB
Python
"""Build a pinned isolated native worker; development VM builds use --backend cpu."""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
from pathlib import Path
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
from runtime_provenance import fingerprint, UPSTREAM_COMMIT
|
||
from macos_runtime import inspect as inspect_macos
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
TAG = "b11461"
|
||
COMMIT = UPSTREAM_COMMIT
|
||
WINDOWS_CRT_FILES = (
|
||
"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll",
|
||
"msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll",
|
||
"vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll",
|
||
)
|
||
LINUX_CPU_VARIANTS = {"x64", "sse42", "sandybridge", "ivybridge", "piledriver", "haswell",
|
||
"skylakex", "cannonlake", "cascadelake", "icelake", "cooperlake", "zen4",
|
||
"alderlake", "sapphirerapids"}
|
||
LINUX_SYSTEM_LIBRARIES = {"libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0",
|
||
"librt.so.1", "libstdc++.so.6", "libgcc_s.so.1", "ld-linux-x86-64.so.2"}
|
||
|
||
|
||
def linux_dispatch(backend):
|
||
"""Only normal Linux Vulkan modes use the packaged shared-module layout."""
|
||
return (sys.platform.startswith("linux") and platform.machine().lower() in {"amd64", "x86_64"}
|
||
and backend in {"auto", "vulkan"})
|
||
|
||
|
||
def elf_dynamic(path):
|
||
"""Read ELF metadata without executing the artifact or following ldd hooks."""
|
||
output = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True)
|
||
values = {}
|
||
for key, value in re.findall(r"\((NEEDED|SONAME|RUNPATH|RPATH)\).*?\[([^\]]+)\]", output):
|
||
values.setdefault(key, []).append(value)
|
||
return values
|
||
|
||
|
||
def bundle_linux_libraries(build, provenance=None):
|
||
"""Flatten approved SONAME files beside the worker and record dependency closure.
|
||
|
||
System libc/C++ ABI remains a declared distro compatibility gate. Vulkan's
|
||
loader is optional: only the dynamically loaded GPU module may depend on it.
|
||
Preflight precedes flattening; the preceding CMake build itself is not a
|
||
transactional release replacement. Staging performs that separate swap.
|
||
"""
|
||
build = Path(build)
|
||
worker = build / "searchi-gemma-worker"
|
||
candidates = [build / "bin" / f"lib{name}.so" for name in ("ggml", "ggml-base", "llama", "mtmd")]
|
||
candidates += [build / f"libggml-cpu-{variant}.so" for variant in sorted(LINUX_CPU_VARIANTS)]
|
||
candidates.append(build / "libggml-vulkan.so")
|
||
records = []
|
||
names = set()
|
||
for link in candidates:
|
||
if not link.is_file():
|
||
raise RuntimeError(f"Required Linux runtime library is missing: {link.name}")
|
||
source = link.resolve()
|
||
if not source.is_relative_to(build.resolve()) or not source.is_file():
|
||
raise RuntimeError("Linux runtime library escapes its build directory")
|
||
metadata = elf_dynamic(source)
|
||
sonames = metadata.get("SONAME", [])
|
||
module = link.parent == build
|
||
if ((not module and len(sonames) != 1) or len(sonames) > 1
|
||
or any(not re.fullmatch(r"lib[A-Za-z0-9_-]+\.so(?:\.\d+)*", name) for name in sonames)):
|
||
raise RuntimeError("Linux runtime has invalid or missing SONAME")
|
||
# Backend discovery uses the unversioned module filename; core linkage
|
||
# uses its SONAME. Both destinations are actual files, never symlinks.
|
||
name = link.name if module else sonames[0]
|
||
if name in names:
|
||
raise RuntimeError("Duplicate Linux runtime library")
|
||
names.add(name)
|
||
destination = build / name
|
||
records.append((source, destination, metadata))
|
||
all_names = names
|
||
for path, metadata in [(worker, elf_dynamic(worker)), *[(destination, metadata) for _, destination, metadata in records]]:
|
||
if metadata.get("RPATH") or metadata.get("RUNPATH") != ["$ORIGIN"]:
|
||
raise RuntimeError(f"Linux runtime requires origin-only RUNPATH: {path.name}")
|
||
needed = set(metadata.get("NEEDED", []))
|
||
allowed = all_names | LINUX_SYSTEM_LIBRARIES
|
||
if path.name == "libggml-vulkan.so":
|
||
allowed = allowed | {"libvulkan.so.1"}
|
||
if needed - allowed:
|
||
raise RuntimeError(f"Unpackaged Linux runtime dependency for {path.name}: {sorted(needed - allowed)}")
|
||
for source, destination, _ in records:
|
||
if destination != source:
|
||
if destination.is_symlink():
|
||
destination.unlink()
|
||
shutil.copy2(source, destination)
|
||
def checksum(path):
|
||
with path.open("rb") as stream:
|
||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||
inventory = {"platform": "linux", "worker_sha256": checksum(worker),
|
||
"build_provenance": provenance if provenance is not None else fingerprint(ROOT),
|
||
"libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)}
|
||
for _, path, _ in sorted(records)]}
|
||
temporary = build / "runtime-libraries.json.partial"
|
||
temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8")
|
||
os.replace(temporary, build / "runtime-libraries.json")
|
||
|
||
|
||
def bundle_windows_crt(build):
|
||
"""Copy unchanged release CRT from the selected Visual Studio installation."""
|
||
cache = (build / "CMakeCache.txt").read_text(encoding="utf-8")
|
||
match = re.search(r"^CMAKE_GENERATOR_INSTANCE:[^=]+=([^\r\n]+)$", cache, re.MULTILINE)
|
||
if not match:
|
||
raise RuntimeError("Selected Visual Studio installation is missing from CMake cache")
|
||
installation = Path(match.group(1))
|
||
auxiliary = installation / "VC/Auxiliary/Build"
|
||
version = (auxiliary / "Microsoft.VCRedistVersion.default.txt").read_text().strip()
|
||
tools_version = (auxiliary / "Microsoft.VCToolsVersion.default.txt").read_text().strip()
|
||
if not re.fullmatch(r"\d+\.\d+\.\d+", version) or version.split(".")[:2] != tools_version.split(".")[:2]:
|
||
raise RuntimeError("Visual Studio CRT redistributable does not match selected build tools")
|
||
cmake_version = [re.search(rf"^CMAKE_CACHE_{part}_VERSION:[^=]+=(\d+)$", cache, re.MULTILINE)
|
||
for part in ("MAJOR", "MINOR", "PATCH")]
|
||
if all(cmake_version):
|
||
compiler_info = build / "CMakeFiles" / ".".join(match.group(1) for match in cmake_version) / "CMakeCXXCompiler.cmake"
|
||
if compiler_info.is_file():
|
||
compiler = compiler_info.read_text(encoding="utf-8")
|
||
actual = re.search(r'set\(CMAKE_CXX_COMPILER_VERSION "(\d+)\.(\d+)\.[^"]+"\)', compiler)
|
||
if not actual or (int(actual.group(1)), int(actual.group(2))) != (19, int(version.split(".")[1])):
|
||
raise RuntimeError("Visual Studio CRT redistributable does not match actual selected compiler")
|
||
source = installation / "VC/Redist/MSVC" / version / "x64/Microsoft.VC143.CRT"
|
||
files = [source / name for name in WINDOWS_CRT_FILES]
|
||
if any(not path.is_file() or path.is_symlink() for path in files):
|
||
raise RuntimeError("Required release Visual Studio CRT redistributable is missing")
|
||
destination = build / "Release"
|
||
for path in files:
|
||
shutil.copy2(path, destination / path.name)
|
||
(destination / "MSVC-REDISTRIBUTION.txt").write_text(
|
||
"Microsoft Visual C++ release runtime — copyright Microsoft Corporation.\n"
|
||
f"Unmodified x64 redistributable files from Visual Studio VC/Redist/MSVC/{version}/x64/Microsoft.VC143.CRT.\n"
|
||
"These proprietary files are distributed under Microsoft's applicable Visual Studio license terms, not searchi's license.\n"
|
||
"Terms: https://visualstudio.microsoft.com/license-terms/vs2022-ga-diagnosticbuildtools/\n"
|
||
"Redistributable list: https://learn.microsoft.com/en-us/visualstudio/releases/2022/redistribution\n"
|
||
"Deployment guidance: https://learn.microsoft.com/en-us/cpp/windows/redistributing-visual-cpp-files\n",
|
||
encoding="utf-8")
|
||
|
||
|
||
def run(arguments, cwd=None):
|
||
subprocess.run([str(value) for value in arguments], cwd=cwd, check=True)
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Build Searchi's pinned Google embedding worker")
|
||
parser.add_argument("--backend", choices=["cpu", "auto", "metal", "vulkan", "cuda"], default="auto")
|
||
parser.add_argument("--jobs", type=int, default=2)
|
||
args = parser.parse_args()
|
||
if not 1 <= args.jobs <= 16:
|
||
parser.error("jobs must be 1–16")
|
||
vulkan_loader = None
|
||
if os.name == "nt" and args.backend in {"auto", "vulkan"}:
|
||
from provision_vulkan import provision
|
||
sdk, vulkan_loader = provision()
|
||
os.environ["VULKAN_SDK"] = str(sdk)
|
||
source = ROOT / ".searchi/references/llama.cpp"
|
||
build = ROOT / ".searchi/runtime" / args.backend
|
||
if not source.exists():
|
||
source.parent.mkdir(parents=True, exist_ok=True)
|
||
run(["git", "clone", "--depth", "1", "--branch", TAG,
|
||
"https://github.com/ggml-org/llama.cpp.git", source])
|
||
actual = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip()
|
||
if actual != COMMIT:
|
||
raise RuntimeError("Runtime reference checkout differs from pinned commit; refusing to reset it")
|
||
patch = Path(__file__).with_name("no-logits.patch")
|
||
check = subprocess.run(["git", "apply", "--check", str(patch)], cwd=source, capture_output=True)
|
||
if check.returncode == 0:
|
||
run(["git", "apply", patch], cwd=source)
|
||
else:
|
||
reverse = subprocess.run(["git", "apply", "--reverse", "--check", str(patch)], cwd=source, capture_output=True)
|
||
if reverse.returncode:
|
||
raise RuntimeError("Runtime memory patch cannot be applied or verified")
|
||
if (os.name == "nt" and args.backend != "cpu") or linux_dispatch(args.backend):
|
||
for patch_name in ["cpu-dispatch.patch", "cpu-only-loader.patch"]:
|
||
dispatch_patch = Path(__file__).with_name(patch_name)
|
||
check = subprocess.run(["git", "apply", "--check", str(dispatch_patch)], cwd=source, capture_output=True)
|
||
if check.returncode == 0:
|
||
run(["git", "apply", dispatch_patch], cwd=source)
|
||
elif subprocess.run(["git", "apply", "--reverse", "--check", str(dispatch_patch)], cwd=source, capture_output=True).returncode:
|
||
raise RuntimeError(f"Runtime patch cannot be applied or verified: {patch_name}")
|
||
cmake_exe = shutil.which("cmake")
|
||
if not cmake_exe:
|
||
try:
|
||
import cmake
|
||
cmake_exe = str(Path(cmake.CMAKE_BIN_DIR) / ("cmake.exe" if os.name == "nt" else "cmake"))
|
||
except ImportError as error:
|
||
raise RuntimeError("Install CMake first (python -m pip install cmake)") from error
|
||
worker_source = ROOT / "crates/searchi-semantic/native"
|
||
provenance = fingerprint(ROOT)
|
||
run([cmake_exe, "-S", worker_source, "-B", build,
|
||
"-DCMAKE_BUILD_TYPE=Release", f"-DLLAMA_CPP_DIR={source}",
|
||
f"-DSEARCHI_ACCELERATION={args.backend}"])
|
||
run([cmake_exe, "--build", build, "--config", "Release", "--parallel", args.jobs])
|
||
if fingerprint(ROOT) != provenance:
|
||
raise RuntimeError("Native build inputs changed during compilation; rerun the build")
|
||
if vulkan_loader:
|
||
for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]:
|
||
shutil.copy2(vulkan_loader / name, build / "Release" / name)
|
||
if os.name == "nt" and args.backend != "cpu":
|
||
directory = build / "Release"
|
||
bundle_windows_crt(build)
|
||
def checksum(path):
|
||
with path.open("rb") as stream:
|
||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||
inventory = {"worker_sha256": checksum(directory / "searchi-gemma-worker.exe"),
|
||
"build_provenance": provenance,
|
||
"libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)}
|
||
for path in sorted(directory.glob("*.dll"))]}
|
||
temporary = directory / "runtime-libraries.json.partial"
|
||
temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8")
|
||
os.replace(temporary, directory / "runtime-libraries.json")
|
||
if linux_dispatch(args.backend):
|
||
bundle_linux_libraries(build, provenance)
|
||
if sys.platform == "darwin" and args.backend in {"auto", "metal"}:
|
||
worker = build / "searchi-gemma-worker"
|
||
inventory = inspect_macos(worker)
|
||
with worker.open("rb") as stream:
|
||
inventory["worker_sha256"] = hashlib.file_digest(stream, "sha256").hexdigest()
|
||
inventory["build_provenance"] = provenance
|
||
temporary = build / "runtime-libraries.json.partial"
|
||
temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8")
|
||
os.replace(temporary, build / "runtime-libraries.json")
|
||
print(f"Runtime build complete: {build}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|