"""CPU-only paired real-worker context startup/memory measurements. No downloads or GPU execution. Windows reports private commitment and resident working set separately; Linux reports PSS/private resident pages. Neither should be summed as model allocations across workers. Requires an idle reference host. """ import argparse import ctypes import hashlib import json import math import os from pathlib import Path import platform import statistics import time from benchmark import ROOT, Session, vectors, require def memory(pid): if os.name == "nt": from ctypes import wintypes class Counters(ctypes.Structure): _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD)] + [(name, ctypes.c_size_t) for name in ["PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage", "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", "QuotaNonPagedPoolUsage", "PagefileUsage", "PeakPagefileUsage", "PrivateUsage"]] kernel = ctypes.WinDLL("kernel32", use_last_error=True) kernel.OpenProcess.restype = wintypes.HANDLE kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] kernel.CloseHandle.argtypes = [wintypes.HANDLE] psapi = ctypes.WinDLL("psapi", use_last_error=True) psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(Counters), wintypes.DWORD] handle = kernel.OpenProcess(0x1000 | 0x10, False, pid) if not handle: raise ctypes.WinError(ctypes.get_last_error()) try: info = Counters() info.cb = ctypes.sizeof(info) if not psapi.GetProcessMemoryInfo(handle, ctypes.byref(info), info.cb): raise ctypes.WinError(ctypes.get_last_error()) return {"private_commit_bytes": info.PrivateUsage, "working_set_bytes": info.WorkingSetSize, "page_fault_count": info.PageFaultCount} finally: kernel.CloseHandle(handle) values = {} for line in Path(f"/proc/{pid}/smaps_rollup").read_text().splitlines(): parts = line.split() if parts[0] in {"Pss:", "Private_Clean:", "Private_Dirty:", "Rss:"}: values[parts[0][:-1] + "_bytes"] = int(parts[1]) * 1024 return values def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--worker", type=Path, required=True) parser.add_argument("--models", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--repeats", type=int, default=3) args = parser.parse_args() if args.output.exists(): parser.error("output must be new; preserve previous evidence") if not 1 <= args.repeats <= 20: parser.error("repeats must be 1..20") manifest = json.loads((ROOT / "tools/embeddinggemma2/model-manifest.json").read_text()) entry = manifest["files"][0] model = args.models / entry["path"] started = time.perf_counter() require(model.stat().st_size == entry["bytes"], 'Benchmark contract failed at query_context_benchmark.py:65') with model.open("rb") as stream: require(hashlib.file_digest(stream, "sha256").hexdigest() == entry["sha256"], 'Benchmark contract failed at query_context_benchmark.py:67') verification_ms = (time.perf_counter() - started) * 1000 samples = [] for repeat in range(args.repeats): # Alternate ordering to expose warm filesystem/cache order effects. for role in (["indexing", "query"] if repeat % 2 == 0 else ["query", "indexing"]): session = Session([str(args.worker.resolve()), "--model", str(model.resolve()), "--threads", "1", "--device", "cpu", "--role", role]) try: hello = session.next() startup_ms = (time.perf_counter() - session.started) * 1000 require(hello["device"] == "cpu" and not hello["vulkan_module_loaded"], 'Benchmark contract failed at query_context_benchmark.py:77') require(hello["role"] == role and hello["context_tokens"] == (512 if role == "query" else 2048), 'Benchmark contract failed at query_context_benchmark.py:78') sample = {"repeat": repeat, "role": role, "hello": hello, "startup_ms": startup_ms, "ready_memory": memory(session.process.pid)} response, first_ms = session.worker({"op": "query", "text": "cancel the supplier partnership", "debug_performance": True}) vectors(response, 1) sample.update(first_query_ms=first_ms, worker=response["performance"], after_query_memory=memory(session.process.pid)) warm = [] # Distinct inputs exercise native query inference without a Rust # query cache. Keep first input separate from these warmed calls. for index in range(20): response, duration = session.worker({"op": "query", "text": f"supplier cancellation notice example {index}", "debug_performance": True}) vectors(response, 1) warm.append({"round_trip_ms": duration, "worker": response["performance"]}) ordered = sorted(item["round_trip_ms"] for item in warm) sample["warm_queries"] = {"samples": warm, "p95_ms": ordered[math.ceil(len(ordered) * .95) - 1]} if role == "query": session.send({"op": "documents", "texts": ["test"], "title": "none"}) rejected = session.next() require(not rejected["ok"] and not rejected["fatal"], 'Benchmark contract failed at query_context_benchmark.py:95') require("only supports" in rejected["error"], 'Benchmark contract failed at query_context_benchmark.py:96') vectors(session.worker({"op": "code_query", "text": "retry failed requests"})[0], 1) samples.append(sample) finally: session.close() report = {"platform": platform.platform(), "cpu_count": os.cpu_count(), "verification_ms": verification_ms, "cache_note": "Cached model bytes; startup excludes Rust verification and desktop preparation. No GPU executed. Repeated fresh processes, not cold disk.", "worker_sha256": hashlib.sha256(args.worker.read_bytes()).hexdigest(), "samples": samples} args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("x", encoding="utf-8") as output: output.write(json.dumps(report, indent=2) + "\n") for role in ["indexing", "query"]: rows = [s for s in samples if s["role"] == role] print(role, "startup median ms", statistics.median(s["startup_ms"] for s in rows), "ready memory", rows[0]["ready_memory"], "after query", rows[0]["after_query_memory"]) if __name__ == "__main__": main()