From d568bdb2b1fd13870505a68b540d44c117dc6dea Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 13:53:03 +0200 Subject: [PATCH] Measure genuine CPU query startup first input and warm latency --- docs/QUERY-CONTEXT.md | 29 +++++++++++++++++++ .../query_context_benchmark.py | 15 +++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/QUERY-CONTEXT.md b/docs/QUERY-CONTEXT.md index 6c51e13..d7821bc 100644 --- a/docs/QUERY-CONTEXT.md +++ b/docs/QUERY-CONTEXT.md @@ -93,3 +93,32 @@ and six paired measurements also passed; see effectively unchanged in those samples. Native macOS and actual GPU context allocation/parity, pressure behavior, long code queries, representative query latency and updated installed desktop packaging remain release checks. + +## Current Windows CPU worker timing, 2026-10-13 + +The normal current worker (`c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73`) +was measured with CPU explicitly selected and one inference thread. Three paired +indexing/query processes alternate order; each retains a separate first request +and twenty distinct warmed inputs with native debug timings. Real 768-dimensional +vectors were checked. Output preservation now refuses existing reports. + +| Query-role repeat | Process startup ms | First inference/IPC ms | Warm p95 ms (20 samples) | +| --- | ---: | ---: | ---: | +| 0 | 729.27 | 100.02 | 71.30 | +| 1 | 662.83 | 106.18 | 78.38 | +| 2 | 704.21 | 100.84 | 71.75 | + +One-time text-model size/SHA verification took 192.45 ms and is separate from +worker startup. Indexing-role startup ranged 624.49–779.14 ms, first query +101.64–110.53 ms, warm p95 73.03–80.47 ms. These are cached-file native-worker +measurements, not cold disk, Rust query-cache, ANN, hybrid retrieval, actual UI +first input or paint. The installed desktop/query worker remained present; +background activity was not controlled, so this is not an idle reference-host +acceptance measurement. All test inference used CPU; no GPU provider was loaded. + +Full ignored report: `.searchi/qa/query-context-warm-current.json`, SHA-256 +`ea7dd0b72eeb1dbd79f06fce92e0475dfdc321d7781e5d06d58fe12755faf0fd`. +The query-role memory observation after first inference was 239,878,144 bytes +private commitment and 274,141,184 bytes resident working set for repeat zero; +these are different metrics and must not be summed. Native desktop first-input +and complete search latency remain separate open checks. diff --git a/tools/embeddinggemma2/query_context_benchmark.py b/tools/embeddinggemma2/query_context_benchmark.py index 6767d35..b9eb746 100644 --- a/tools/embeddinggemma2/query_context_benchmark.py +++ b/tools/embeddinggemma2/query_context_benchmark.py @@ -8,6 +8,7 @@ import argparse import ctypes import hashlib import json +import math import os from pathlib import Path import platform @@ -53,6 +54,8 @@ def main(): parser.add_argument("--output", type=Path, required=True) parser.add_argument("--repeats", type=int, default=3) args = parser.parse_args() + if args.output.exists(): + parser.error("output must be new; preserve previous evidence") if not 1 <= args.repeats <= 20: parser.error("repeats must be 1..20") manifest = json.loads((ROOT / "tools/embeddinggemma2/model-manifest.json").read_text()) @@ -77,6 +80,15 @@ def main(): response, first_ms = session.worker({"op": "query", "text": "cancel the supplier partnership", "debug_performance": True}) vectors(response, 1) sample.update(first_query_ms=first_ms, worker=response["performance"], after_query_memory=memory(session.process.pid)) + warm = [] + # Distinct inputs exercise native query inference without a Rust + # query cache. Keep first input separate from these warmed calls. + for index in range(20): + response, duration = session.worker({"op": "query", "text": f"supplier cancellation notice example {index}", "debug_performance": True}) + vectors(response, 1) + warm.append({"round_trip_ms": duration, "worker": response["performance"]}) + ordered = sorted(item["round_trip_ms"] for item in warm) + sample["warm_queries"] = {"samples": warm, "p95_ms": ordered[math.ceil(len(ordered) * .95) - 1]} if role == "query": session.send({"op": "documents", "texts": ["test"], "title": "none"}) rejected = session.next() @@ -88,7 +100,8 @@ def main(): session.close() report = {"platform": platform.platform(), "cpu_count": os.cpu_count(), "verification_ms": verification_ms, "cache_note": "Cached model bytes; startup excludes Rust verification and desktop preparation. No GPU executed. Repeated fresh processes, not cold disk.", "worker_sha256": hashlib.sha256(args.worker.read_bytes()).hexdigest(), "samples": samples} args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + with args.output.open("x", encoding="utf-8") as output: + output.write(json.dumps(report, indent=2) + "\n") for role in ["indexing", "query"]: rows = [s for s in samples if s["role"] == role] print(role, "startup median ms", statistics.median(s["startup_ms"] for s in rows), "ready memory", rows[0]["ready_memory"], "after query", rows[0]["after_query_memory"])