Verify extracted Windows CLI through actual MCP search and citations

2026-10-13 11:57:08 +02:00
1 parent 2be6b3d2b4
commit 3a336ca948
@@ -0,0 +1,129 @@
+"""Exercise an extracted Windows CLI's read-only MCP boundary on owned fixtures."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import queue
+import subprocess
+import threading
+import time
+
+
+def check(condition, message):
+    if not condition:
+        raise RuntimeError(message)
+
+
+def smoke(cli, work):
+    cli = cli.resolve(strict=True)
+    work.mkdir(parents=True, exist_ok=False)
+    corpus = work / "documents"
+    corpus.mkdir()
+    (corpus / "contract.txt").write_text(
+        "Termination clause: sixty days written notice to end the supplier partnership.",
+        encoding="utf-8",
+    )
+    (corpus / "retry.rs").write_text(
+        "pub fn refreshAccessToken() { /* retry failed requests */ }", encoding="utf-8"
+    )
+    (corpus / "symbol-note.txt").write_text("refreshAccessToken", encoding="utf-8")
+    db = work / "index.sqlite"
+    environment = dict(os.environ, SEARCHI_FORCE_CPU="1")
+    subprocess.run(
+        [str(cli), "index", "--db", str(db), "--root", str(corpus)],
+        env=environment, check=True, capture_output=True, timeout=60,
+    )
+    child = subprocess.Popen(
+        [str(cli), "mcp", "--db", str(db)], env=environment,
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+        text=True, encoding="utf-8",
+    )
+    output = queue.Queue(maxsize=128)
+
+    def reader():
+        for line in child.stdout:
+            try:
+                output.put(line, timeout=1)
+            except queue.Full:
+                return
+        output.put(None, timeout=1)
+
+    reading = threading.Thread(target=reader, daemon=True)
+    reading.start()
+
+    def send(value):
+        child.stdin.write(json.dumps(value) + "\n")
+        child.stdin.flush()
+
+    def request(identifier, method, params):
+        send({"jsonrpc": "2.0", "id": identifier, "method": method, "params": params})
+        deadline = time.monotonic() + 15
+        for _ in range(128):
+            remaining = deadline - time.monotonic()
+            check(remaining > 0, "MCP response deadline exceeded")
+            line = output.get(timeout=remaining)
+            check(line is not None, "MCP terminated before responding")
+            response = json.loads(line)
+            check(response.get("jsonrpc") == "2.0", "Invalid JSON-RPC output")
+            if response.get("id") == identifier:
+                check("error" not in response, "MCP protocol error")
+                return response["result"]
+        raise RuntimeError("MCP notification ceiling exceeded")
+
+    try:
+        hello = request(1, "initialize", {
+            "protocolVersion": "2025-11-25", "capabilities": {},
+            "clientInfo": {"name": "searchi-package-smoke", "version": "1"},
+        })
+        check(hello["serverInfo"]["name"] == "searchi", "Wrong MCP server")
+        send({"jsonrpc": "2.0", "method": "notifications/initialized"})
+        tools = request(2, "tools/list", {})["tools"]
+        check({tool["name"] for tool in tools} == {"search", "stats", "read_source"},
+              "Unexpected tools exposed")
+        check(all(tool["annotations"]["readOnlyHint"] for tool in tools),
+              "Missing read-only annotation")
+        found = request(3, "tools/call", {"name": "search", "arguments": {
+            "query": "termination", "extension": "txt", "limit": 3,
+        }})["structuredContent"]
+        check(found["mode"] == "lexical_only", "Incorrect readiness mode")
+        check(found["results"][0]["filename"] == "contract.txt", "Wrong document")
+        source = request(4, "tools/call", {"name": "read_source", "arguments": {
+            "handle": found["results"][0]["source_handle"],
+        }})["structuredContent"]
+        check("sixty days" in source["text"], "Missing cited source passage")
+        invented = request(5, "tools/call", {"name": "read_source", "arguments": {
+            "handle": "invented",
+        }})
+        check(invented.get("isError") is True, "Invented source handle accepted")
+        code = request(6, "tools/call", {"name": "search", "arguments": {
+            "query": "refreshAccessToken", "code_only": True,
+        }})["structuredContent"]
+        check([row["filename"] for row in code["results"]] == ["retry.rs"],
+              "Code filter included ordinary documents")
+        with cli.open("rb") as stream:
+            checksum = hashlib.file_digest(stream, "sha256").hexdigest()
+        report = {"complete": True, "cli_sha256": checksum, "mode": "lexical_only",
+                  "cpu_forced": True, "inference": False, "tool_names": [t["name"] for t in tools],
+                  "search": found, "source": source, "code": code}
+        with (work / "report.json").open("x", encoding="utf-8") as stream:
+            json.dump(report, stream, indent=2)
+        return report
+    finally:
+        # Only this subprocess is owned; leave the installed desktop untouched.
+        child.kill()
+        child.wait(timeout=15)
+        reading.join(timeout=2)
+        child.stdin.close()
+        child.stdout.close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--cli", type=Path, required=True)
+    parser.add_argument("--work-dir", type=Path, required=True,
+                        help="New owned fixture/evidence directory, retained after the run")
+    args = parser.parse_args()
+    result = smoke(args.cli, args.work_dir.resolve())
+    print(json.dumps({"complete": result["complete"], "mode": result["mode"],
+                      "cli_sha256": result["cli_sha256"]}))