feat: add Google code retrieval across desktop CLI and MCP

tools/code-eval/evaluate.py

@@ -0,0 +1,96 @@
+"""Frozen tiny code-retrieval regression; genuine Google worker, CPU only.
+
+Generated public fixtures, not a production accuracy benchmark. Run from repo
+after rebuilding CLI and CPU worker. No private folders are indexed.
+"""
+import hashlib
+from datetime import datetime, timezone
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("benchmark", ROOT / "tools/embeddinggemma2/benchmark.py")
+BENCH = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(BENCH)
+
+FILES = {
+    "a.py": "import hashlib\ndef process(path):\n    h = hashlib.sha256()\n    with open(path, 'rb') as stream:\n        for chunk in iter(lambda: stream.read(65536), b''):\n            h.update(chunk)\n    return h.hexdigest()\n",
+    "b.cpp": "#include <vector>\nint locate(const std::vector<int>& v, int x) {\n  int lo = 0, hi = v.size();\n  while (lo < hi) {\n    int mid = lo + (hi-lo)/2;\n    if (v[mid] < x) lo = mid+1; else hi = mid;\n  }\n  return lo;\n}\n",
+    "c.ts": "export function process(rows: {category: string; amount: number}[]) {\n  const totals = new Map<string, number>();\n  for (const row of rows) totals.set(row.category, (totals.get(row.category) ?? 0) + row.amount);\n  return totals;\n}\n",
+    "d.py": "import time\nimport urllib.request\ndef process(url):\n    for attempt in range(5):\n        try:\n            return urllib.request.urlopen(url, timeout=10).read()\n        except OSError:\n            if attempt == 4: raise\n            time.sleep(0.2 * 2 ** attempt)\n",
+    "e.cs": "using System;\nusing System.Threading;\nusing System.Threading.Tasks;\npublic static class Module {\n public static async Task Process(SemaphoreSlim gate, Func<Task> work) {\n  await gate.WaitAsync();\n  try { await work(); } finally { gate.Release(); }\n }\n}\n",
+    "f.php": "<?php\nfunction process($value) {\n    return htmlspecialchars($value, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8');\n}\n",
+    "g.py": "def process(rows):\n    return sorted(rows, key=lambda row: row['price'], reverse=True)[:10]\n",
+    "h.rs": "fn refreshAccessToken(previous: &str) -> String {\n    format!(\"renewed:{}\", previous)\n}\n",
+    "notes.txt": "refreshAccessToken retry failed requests with exponential backoff compute a file digest sum amounts by category escape HTML sorted numbers",
+}
+QUERIES = [
+    ("calculate a file's SHA256 without loading the entire file into memory", "a.py"),
+    ("find the first position where sorted numbers are greater than or equal to a target", "b.cpp"),
+    ("sum transaction amounts separately for each category", "c.ts"),
+    ("retry failed requests with exponential backoff", "d.py"),
+    ("release a semaphore even if the asynchronous operation throws", "e.cs"),
+    ("escape unsafe HTML characters before displaying user input", "f.php"),
+    ("return the ten most expensive records", "g.py"),
+    ("Datei prüfen ohne sie vollständig in den Speicher zu laden", "a.py"),
+    ("refreshAccessToken", "h.rs"),
+]
+
+
+def main():
+    output = ROOT / ".searchi/code-eval"
+    corpus = output / "corpus"
+    corpus.mkdir(parents=True, exist_ok=True)
+    for name, content in FILES.items():
+        (corpus / name).write_text(content, encoding="utf-8", newline="\n")
+    cli = ROOT / "target/release" / ("searchi.exe" if os.name == "nt" else "searchi")
+    models = ROOT / ".searchi/models/embeddinggemma2"
+    worker = ROOT / ".searchi/runtime/cpu/Release" / ("searchi-gemma-worker.exe" if os.name == "nt" else "searchi-gemma-worker")
+    db = output / "index.db"
+    environment = dict(os.environ, SEARCHI_FORCE_CPU="1")
+    def run(*args):
+        subprocess.run([str(cli), *map(str, args)], env=environment, check=True)
+    run("index", "--db", db, "--root", corpus)
+    run("embed", "--db", db, "--model-dir", models, "--embedding-worker", worker, "--text-only")
+    rows = []
+    for mode in ("lexical_only", "hybrid"):
+        command = [str(cli), "mcp", "--db", str(db)]
+        if mode == "hybrid":
+            command += ["--model-dir", str(models), "--embedding-worker", str(worker)]
+        session = BENCH.Session(command)
+        try:
+            session.mcp(1, "initialize", {"protocolVersion": "2025-03-26", "capabilities": {}, "clientInfo": {"name": "code-eval", "version": "1"}})
+            session.send({"jsonrpc": "2.0", "method": "notifications/initialized"})
+            for number, (query, expected) in enumerate(QUERIES, 2):
+                result, duration = session.mcp(number, "tools/call", {"name": "search", "arguments": {"query": query, "code_only": True, "limit": 10}})
+                assert not result.get("isError"), result
+                data = result["structuredContent"]
+                assert data["mode"] == mode and data["code_only"]
+                names = [r["filename"] for r in data["results"]]
+                assert "notes.txt" not in names
+                rank = names.index(expected) + 1 if expected in names else None
+                source_verified = False
+                if rank is not None:
+                    matched = data["results"][rank - 1]
+                    source, _ = session.mcp(number + 100, "tools/call", {"name": "read_source", "arguments": {"handle": matched["source_handle"]}})
+                    assert not source.get("isError"), source
+                    source = source["structuredContent"]
+                    assert source["source"] == {"kind": "line", "number": 1}
+                    assert source["text"] == FILES[expected]
+                    source_verified = True
+                rows.append({"mode": mode, "query": query, "expected": expected, "rank": rank, "results": names, "roundtrip_ms": duration, "source_verified": source_verified})
+        finally:
+            session.close()
+    report = {"recorded_at": datetime.now(timezone.utc).isoformat(), "device": "cpu", "cpu_threads": 1, "model": "google/embeddinggemma-2", "precision": "verified Q8_0 weights, CPU inference", "query_task": "task: code retrieval | query: ", "scope": "8 tiny source files and 1 ordinary-document hard negative; synthetic regression, not general code accuracy", "fixture_sha256": {n: hashlib.sha256(c.encode()).hexdigest() for n, c in FILES.items()}, "queries": rows}
+    (output / "results.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    for mode in ("lexical_only", "hybrid"):
+        selected = [r for r in rows if r["mode"] == mode]
+        print(mode, "top1", sum(r["rank"] == 1 for r in selected), "/", len(selected), "top3", sum(r["rank"] is not None and r["rank"] <= 3 for r in selected))
+    print("Recorded", output / "results.json")
+
+
+if __name__ == "__main__":
+    main()