"""Build a pinned isolated native worker; development VM builds use --backend cpu.""" import argparse import hashlib import json import os import platform import re from pathlib import Path import shutil import subprocess import sys from runtime_provenance import fingerprint, UPSTREAM_COMMIT from macos_runtime import inspect as inspect_macos ROOT = Path(__file__).resolve().parents[2] TAG = "b11461" COMMIT = UPSTREAM_COMMIT WINDOWS_CRT_FILES = ( "concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll", ) LINUX_CPU_VARIANTS = {"x64", "sse42", "sandybridge", "ivybridge", "piledriver", "haswell", "skylakex", "cannonlake", "cascadelake", "icelake", "cooperlake", "zen4", "alderlake", "sapphirerapids"} LINUX_SYSTEM_LIBRARIES = {"libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0", "librt.so.1", "libstdc++.so.6", "libgcc_s.so.1", "ld-linux-x86-64.so.2"} def linux_dispatch(backend): """Only normal Linux Vulkan modes use the packaged shared-module layout.""" return (sys.platform.startswith("linux") and platform.machine().lower() in {"amd64", "x86_64"} and backend in {"auto", "vulkan"}) def elf_dynamic(path): """Read ELF metadata without executing the artifact or following ldd hooks.""" output = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True) values = {} for key, value in re.findall(r"\((NEEDED|SONAME|RUNPATH|RPATH)\).*?\[([^\]]+)\]", output): values.setdefault(key, []).append(value) return values def bundle_linux_libraries(build, provenance=None): """Flatten approved SONAME files beside the worker and record dependency closure. System libc/C++ ABI remains a declared distro compatibility gate. Vulkan's loader is optional: only the dynamically loaded GPU module may depend on it. Preflight precedes flattening; the preceding CMake build itself is not a transactional release replacement. Staging performs that separate swap. """ build = Path(build) worker = build / "searchi-gemma-worker" candidates = [build / "bin" / f"lib{name}.so" for name in ("ggml", "ggml-base", "llama", "mtmd")] candidates += [build / f"libggml-cpu-{variant}.so" for variant in sorted(LINUX_CPU_VARIANTS)] candidates.append(build / "libggml-vulkan.so") records = [] names = set() for link in candidates: if not link.is_file(): raise RuntimeError(f"Required Linux runtime library is missing: {link.name}") source = link.resolve() if not source.is_relative_to(build.resolve()) or not source.is_file(): raise RuntimeError("Linux runtime library escapes its build directory") metadata = elf_dynamic(source) sonames = metadata.get("SONAME", []) module = link.parent == build if ((not module and len(sonames) != 1) or len(sonames) > 1 or any(not re.fullmatch(r"lib[A-Za-z0-9_-]+\.so(?:\.\d+)*", name) for name in sonames)): raise RuntimeError("Linux runtime has invalid or missing SONAME") # Backend discovery uses the unversioned module filename; core linkage # uses its SONAME. Both destinations are actual files, never symlinks. name = link.name if module else sonames[0] if name in names: raise RuntimeError("Duplicate Linux runtime library") names.add(name) destination = build / name records.append((source, destination, metadata)) all_names = names for path, metadata in [(worker, elf_dynamic(worker)), *[(destination, metadata) for _, destination, metadata in records]]: if metadata.get("RPATH") or metadata.get("RUNPATH") != ["$ORIGIN"]: raise RuntimeError(f"Linux runtime requires origin-only RUNPATH: {path.name}") needed = set(metadata.get("NEEDED", [])) allowed = all_names | LINUX_SYSTEM_LIBRARIES if path.name == "libggml-vulkan.so": allowed = allowed | {"libvulkan.so.1"} if needed - allowed: raise RuntimeError(f"Unpackaged Linux runtime dependency for {path.name}: {sorted(needed - allowed)}") for source, destination, _ in records: if destination != source: if destination.is_symlink(): destination.unlink() shutil.copy2(source, destination) def checksum(path): with path.open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest() inventory = {"platform": "linux", "worker_sha256": checksum(worker), "build_provenance": provenance if provenance is not None else fingerprint(ROOT), "libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)} for _, path, _ in sorted(records)]} temporary = build / "runtime-libraries.json.partial" temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8") os.replace(temporary, build / "runtime-libraries.json") def bundle_windows_crt(build): """Copy unchanged release CRT from the selected Visual Studio installation.""" cache = (build / "CMakeCache.txt").read_text(encoding="utf-8") match = re.search(r"^CMAKE_GENERATOR_INSTANCE:[^=]+=([^\r\n]+)$", cache, re.MULTILINE) if not match: raise RuntimeError("Selected Visual Studio installation is missing from CMake cache") installation = Path(match.group(1)) auxiliary = installation / "VC/Auxiliary/Build" version = (auxiliary / "Microsoft.VCRedistVersion.default.txt").read_text().strip() tools_version = (auxiliary / "Microsoft.VCToolsVersion.default.txt").read_text().strip() if not re.fullmatch(r"\d+\.\d+\.\d+", version) or version.split(".")[:2] != tools_version.split(".")[:2]: raise RuntimeError("Visual Studio CRT redistributable does not match selected build tools") cmake_version = [re.search(rf"^CMAKE_CACHE_{part}_VERSION:[^=]+=(\d+)$", cache, re.MULTILINE) for part in ("MAJOR", "MINOR", "PATCH")] if all(cmake_version): compiler_info = build / "CMakeFiles" / ".".join(match.group(1) for match in cmake_version) / "CMakeCXXCompiler.cmake" if compiler_info.is_file(): compiler = compiler_info.read_text(encoding="utf-8") actual = re.search(r'set\(CMAKE_CXX_COMPILER_VERSION "(\d+)\.(\d+)\.[^"]+"\)', compiler) if not actual or (int(actual.group(1)), int(actual.group(2))) != (19, int(version.split(".")[1])): raise RuntimeError("Visual Studio CRT redistributable does not match actual selected compiler") source = installation / "VC/Redist/MSVC" / version / "x64/Microsoft.VC143.CRT" files = [source / name for name in WINDOWS_CRT_FILES] if any(not path.is_file() or path.is_symlink() for path in files): raise RuntimeError("Required release Visual Studio CRT redistributable is missing") destination = build / "Release" for path in files: shutil.copy2(path, destination / path.name) (destination / "MSVC-REDISTRIBUTION.txt").write_text( "Microsoft Visual C++ release runtime — copyright Microsoft Corporation.\n" f"Unmodified x64 redistributable files from Visual Studio VC/Redist/MSVC/{version}/x64/Microsoft.VC143.CRT.\n" "These proprietary files are distributed under Microsoft's applicable Visual Studio license terms, not searchi's license.\n" "Terms: https://visualstudio.microsoft.com/license-terms/vs2022-ga-diagnosticbuildtools/\n" "Redistributable list: https://learn.microsoft.com/en-us/visualstudio/releases/2022/redistribution\n" "Deployment guidance: https://learn.microsoft.com/en-us/cpp/windows/redistributing-visual-cpp-files\n", encoding="utf-8") def run(arguments, cwd=None): subprocess.run([str(value) for value in arguments], cwd=cwd, check=True) def main(): parser = argparse.ArgumentParser(description="Build Searchi's pinned Google embedding worker") parser.add_argument("--backend", choices=["cpu", "auto", "metal", "vulkan", "cuda"], default="auto") parser.add_argument("--jobs", type=int, default=2) args = parser.parse_args() if not 1 <= args.jobs <= 16: parser.error("jobs must be 1–16") vulkan_loader = None if os.name == "nt" and args.backend in {"auto", "vulkan"}: from provision_vulkan import provision sdk, vulkan_loader = provision() os.environ["VULKAN_SDK"] = str(sdk) source = ROOT / ".searchi/references/llama.cpp" build = ROOT / ".searchi/runtime" / args.backend if not source.exists(): source.parent.mkdir(parents=True, exist_ok=True) run(["git", "clone", "--depth", "1", "--branch", TAG, "https://github.com/ggml-org/llama.cpp.git", source]) actual = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() if actual != COMMIT: raise RuntimeError("Runtime reference checkout differs from pinned commit; refusing to reset it") patch = Path(__file__).with_name("no-logits.patch") check = subprocess.run(["git", "apply", "--check", str(patch)], cwd=source, capture_output=True) if check.returncode == 0: run(["git", "apply", patch], cwd=source) else: reverse = subprocess.run(["git", "apply", "--reverse", "--check", str(patch)], cwd=source, capture_output=True) if reverse.returncode: raise RuntimeError("Runtime memory patch cannot be applied or verified") if (os.name == "nt" and args.backend != "cpu") or linux_dispatch(args.backend): for patch_name in ["cpu-dispatch.patch", "cpu-only-loader.patch"]: dispatch_patch = Path(__file__).with_name(patch_name) check = subprocess.run(["git", "apply", "--check", str(dispatch_patch)], cwd=source, capture_output=True) if check.returncode == 0: run(["git", "apply", dispatch_patch], cwd=source) elif subprocess.run(["git", "apply", "--reverse", "--check", str(dispatch_patch)], cwd=source, capture_output=True).returncode: raise RuntimeError(f"Runtime patch cannot be applied or verified: {patch_name}") cmake_exe = shutil.which("cmake") if not cmake_exe: try: import cmake cmake_exe = str(Path(cmake.CMAKE_BIN_DIR) / ("cmake.exe" if os.name == "nt" else "cmake")) except ImportError as error: raise RuntimeError("Install CMake first (python -m pip install cmake)") from error worker_source = ROOT / "crates/searchi-semantic/native" provenance = fingerprint(ROOT) run([cmake_exe, "-S", worker_source, "-B", build, "-DCMAKE_BUILD_TYPE=Release", f"-DLLAMA_CPP_DIR={source}", f"-DSEARCHI_ACCELERATION={args.backend}"]) run([cmake_exe, "--build", build, "--config", "Release", "--parallel", args.jobs]) if fingerprint(ROOT) != provenance: raise RuntimeError("Native build inputs changed during compilation; rerun the build") if vulkan_loader: for name in ["vulkan-1.dll", "VULKAN-LICENSE.txt"]: shutil.copy2(vulkan_loader / name, build / "Release" / name) if os.name == "nt" and args.backend != "cpu": directory = build / "Release" bundle_windows_crt(build) def checksum(path): with path.open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest() inventory = {"worker_sha256": checksum(directory / "searchi-gemma-worker.exe"), "build_provenance": provenance, "libraries": [{"path": path.name, "bytes": path.stat().st_size, "sha256": checksum(path)} for path in sorted(directory.glob("*.dll"))]} temporary = directory / "runtime-libraries.json.partial" temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8") os.replace(temporary, directory / "runtime-libraries.json") if linux_dispatch(args.backend): bundle_linux_libraries(build, provenance) if sys.platform == "darwin" and args.backend in {"auto", "metal"}: worker = build / "searchi-gemma-worker" inventory = inspect_macos(worker) with worker.open("rb") as stream: inventory["worker_sha256"] = hashlib.file_digest(stream, "sha256").hexdigest() inventory["build_provenance"] = provenance temporary = build / "runtime-libraries.json.partial" temporary.write_text(json.dumps(inventory, indent=2), encoding="utf-8") os.replace(temporary, build / "runtime-libraries.json") print(f"Runtime build complete: {build}") if __name__ == "__main__": main()