From 3a336ca948474caa098b5b1dc6a0f4329ed9bb3a Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 11:57:08 +0200 Subject: [PATCH] Verify extracted Windows CLI through actual MCP search and citations --- docs/AGENT-INTEGRATION.md | 20 ++++++ tools/windows_agent_smoke.py | 129 +++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 tools/windows_agent_smoke.py diff --git a/docs/AGENT-INTEGRATION.md b/docs/AGENT-INTEGRATION.md index eb52ec7..47a5adc 100644 --- a/docs/AGENT-INTEGRATION.md +++ b/docs/AGENT-INTEGRATION.md @@ -35,6 +35,26 @@ CLI SHA-256: The archive is local only; the Gitea attachment size limit previously rejected even the smaller desktop archive. Published installer contents are unchanged. +### Extracted CLI MCP verification + +`tools/windows_agent_smoke.py` additionally executes the extracted CLI against +a new owned three-file corpus. It checks the real stdio handshake, the exact +three read-only tools, document search and cited passage, rejection of invented +source handles, and exclusion of an ordinary document containing the same code +symbol under `code_only`. Each response has a 15-second deadline; only the +owned MCP subprocess is terminated. The fixture/evidence directory is retained +and an existing directory is never reused or deleted. + +Both normal and optimized Python runs passed on the Windows VM, 2026-10-13, +using the archive CLI hash above. This validates actual lexical MCP behavior +from the package, not semantic readiness, media helpers or GPU execution. +Optimized-run evidence: `.searchi/qa/agent-mcp-smoke-optimized-20261013/report.json`, +SHA-256 `fd5cf6244410b2373ffac38cf94e9f25b7c711737401ee96d6cb29721a1f8e51`. + +```powershell +python tools/windows_agent_smoke.py --cli C:/path/to/searchi-cli.exe --work-dir .searchi/qa/new-agent-smoke +``` + For names/content only: ```sh diff --git a/tools/windows_agent_smoke.py b/tools/windows_agent_smoke.py new file mode 100644 index 0000000..ee225b2 --- /dev/null +++ b/tools/windows_agent_smoke.py @@ -0,0 +1,129 @@ +"""Exercise an extracted Windows CLI's read-only MCP boundary on owned fixtures.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import queue +import subprocess +import threading +import time + + +def check(condition, message): + if not condition: + raise RuntimeError(message) + + +def smoke(cli, work): + cli = cli.resolve(strict=True) + work.mkdir(parents=True, exist_ok=False) + corpus = work / "documents" + corpus.mkdir() + (corpus / "contract.txt").write_text( + "Termination clause: sixty days written notice to end the supplier partnership.", + encoding="utf-8", + ) + (corpus / "retry.rs").write_text( + "pub fn refreshAccessToken() { /* retry failed requests */ }", encoding="utf-8" + ) + (corpus / "symbol-note.txt").write_text("refreshAccessToken", encoding="utf-8") + db = work / "index.sqlite" + environment = dict(os.environ, SEARCHI_FORCE_CPU="1") + subprocess.run( + [str(cli), "index", "--db", str(db), "--root", str(corpus)], + env=environment, check=True, capture_output=True, timeout=60, + ) + child = subprocess.Popen( + [str(cli), "mcp", "--db", str(db)], env=environment, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, encoding="utf-8", + ) + output = queue.Queue(maxsize=128) + + def reader(): + for line in child.stdout: + try: + output.put(line, timeout=1) + except queue.Full: + return + output.put(None, timeout=1) + + reading = threading.Thread(target=reader, daemon=True) + reading.start() + + def send(value): + child.stdin.write(json.dumps(value) + "\n") + child.stdin.flush() + + def request(identifier, method, params): + send({"jsonrpc": "2.0", "id": identifier, "method": method, "params": params}) + deadline = time.monotonic() + 15 + for _ in range(128): + remaining = deadline - time.monotonic() + check(remaining > 0, "MCP response deadline exceeded") + line = output.get(timeout=remaining) + check(line is not None, "MCP terminated before responding") + response = json.loads(line) + check(response.get("jsonrpc") == "2.0", "Invalid JSON-RPC output") + if response.get("id") == identifier: + check("error" not in response, "MCP protocol error") + return response["result"] + raise RuntimeError("MCP notification ceiling exceeded") + + try: + hello = request(1, "initialize", { + "protocolVersion": "2025-11-25", "capabilities": {}, + "clientInfo": {"name": "searchi-package-smoke", "version": "1"}, + }) + check(hello["serverInfo"]["name"] == "searchi", "Wrong MCP server") + send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + tools = request(2, "tools/list", {})["tools"] + check({tool["name"] for tool in tools} == {"search", "stats", "read_source"}, + "Unexpected tools exposed") + check(all(tool["annotations"]["readOnlyHint"] for tool in tools), + "Missing read-only annotation") + found = request(3, "tools/call", {"name": "search", "arguments": { + "query": "termination", "extension": "txt", "limit": 3, + }})["structuredContent"] + check(found["mode"] == "lexical_only", "Incorrect readiness mode") + check(found["results"][0]["filename"] == "contract.txt", "Wrong document") + source = request(4, "tools/call", {"name": "read_source", "arguments": { + "handle": found["results"][0]["source_handle"], + }})["structuredContent"] + check("sixty days" in source["text"], "Missing cited source passage") + invented = request(5, "tools/call", {"name": "read_source", "arguments": { + "handle": "invented", + }}) + check(invented.get("isError") is True, "Invented source handle accepted") + code = request(6, "tools/call", {"name": "search", "arguments": { + "query": "refreshAccessToken", "code_only": True, + }})["structuredContent"] + check([row["filename"] for row in code["results"]] == ["retry.rs"], + "Code filter included ordinary documents") + with cli.open("rb") as stream: + checksum = hashlib.file_digest(stream, "sha256").hexdigest() + report = {"complete": True, "cli_sha256": checksum, "mode": "lexical_only", + "cpu_forced": True, "inference": False, "tool_names": [t["name"] for t in tools], + "search": found, "source": source, "code": code} + with (work / "report.json").open("x", encoding="utf-8") as stream: + json.dump(report, stream, indent=2) + return report + finally: + # Only this subprocess is owned; leave the installed desktop untouched. + child.kill() + child.wait(timeout=15) + reading.join(timeout=2) + child.stdin.close() + child.stdout.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--work-dir", type=Path, required=True, + help="New owned fixture/evidence directory, retained after the run") + args = parser.parse_args() + result = smoke(args.cli, args.work_dir.resolve()) + print(json.dumps({"complete": result["complete"], "mode": result["mode"], + "cli_sha256": result["cli_sha256"]}))