From 81846b3e98c64446e02a61aefef6a9908d466686 Mon Sep 17 00:00:00 2001 From: karl Date: Mon, 12 Oct 2026 21:22:11 +0200 Subject: [PATCH] feat: add Google code retrieval across desktop CLI and MCP --- AGENTS.md | 4 + README.md | 17 +- apps/desktop/src-tauri/src/main.rs | 50 ++++ apps/desktop/src/main.ts | 19 +- apps/desktop/src/match-display.ts | 4 +- apps/desktop/src/search-state.ts | 13 + apps/desktop/src/style.css | 12 + apps/desktop/tests/match-display.test.mjs | 5 + apps/desktop/tests/search-state.test.mjs | 31 ++- crates/searchi-cli/src/main.rs | 5 + crates/searchi-cli/src/mcp.rs | 5 + crates/searchi-cli/tests/commands.rs | 52 ++++ crates/searchi-cli/tests/mcp.rs | 25 ++ crates/searchi-core/src/code.rs | 52 ++++ crates/searchi-core/src/extraction.rs | 65 +++-- crates/searchi-core/src/lib.rs | 25 +- crates/searchi-core/tests/code_search.rs | 92 +++++++ crates/searchi-semantic/native/worker.cpp | 52 +++- crates/searchi-semantic/src/gemma.rs | 32 ++- crates/searchi-semantic/tests/code_cpu.rs | 66 +++++ crates/searchi-service/src/lib.rs | 45 +++- crates/searchi-service/src/routing.rs | 9 +- crates/searchi-service/src/store.rs | 83 +++++- crates/searchi-service/tests/query_cache.rs | 34 +++ docs/CODE-QUALITY.md | 78 ++++++ docs/CODE-SEARCH.md | 80 ++++++ docs/ROADMAP.md | 19 ++ docs/VALIDATION.md | 17 +- docs/evidence/code-search-cpu.json | 267 ++++++++++++++++++++ skills/searchi-local-search/SKILL.md | 6 + tools/code-eval/evaluate.py | 96 +++++++ tools/desktop-capture/backend/src/main.rs | 2 + 32 files changed, 1303 insertions(+), 59 deletions(-) create mode 100644 crates/searchi-core/src/code.rs create mode 100644 crates/searchi-core/tests/code_search.rs create mode 100644 crates/searchi-semantic/tests/code_cpu.rs create mode 100644 docs/CODE-QUALITY.md create mode 100644 docs/CODE-SEARCH.md create mode 100644 docs/evidence/code-search-cpu.json create mode 100644 tools/code-eval/evaluate.py diff --git a/AGENTS.md b/AGENTS.md index e8b800b..01c0a4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,10 @@ readiness without completed release gates and recorded evidence. - Use bounded work, explicit errors, versioned storage, and offline defaults. - Never substitute simulated embeddings for working semantic search. - Production semantic search must use Google EmbeddingGemma 2 as in DigUp. + Code scope uses Google's CodeRetrieval query prefix; query-vector cache keys + must distinguish code and general search tasks. Preserve exact source bytes, + line locations and code filtering before all candidate limits. Re-run actual + CPU code fixtures after changes to tokenizer offsets or code extraction. MiniLM proof artifacts are historical experiments, not an accepted backend. - The desktop UI is the primary product, including local model setup and semantic indexing/search. Keep a separate CLI-only build for headless Linux with no GUI dependencies. diff --git a/README.md b/README.md index 72515c7..5315b69 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

searchi

Find what matters. Locally.

-

Your documents, photos, music and movies. One place to search.

+

Your documents, code, photos, music and movies. One place to search.

You remember what was in the file. searchi helps you find it—even when you don't remember what it was called. @@ -25,6 +25,8 @@ termination passage into view.* | The file's name | `Supplier agreement.pdf` | | Words inside a document | `sixty days written notice` | | What a document is about | `cancel the supplier partnership` | +| What a piece of code does | `retry failed requests with exponential backoff` | +| An exact code symbol | `refreshAccessToken` | | A photo's subject | `mountains reflected in calm water` | | An artist or song title | `Nina Simone` or `Feeling Good`, if present in your music tags | | A scene or sound | Try `rain and thunder` or `a boat crossing a lake` in experimental media search | @@ -37,6 +39,19 @@ sound. Use **All files** to narrow results by folder, file extension, modification date, file size or media duration. Choose a result to inspect it before opening. +### Find the implementation + +Choose **Code** in the **All files** filters to search source files. Look up a +symbol you know, or describe the behavior you need. Local Google embeddings +help connect your description to an implementation; the preview preserves code +formatting and shows the source passage's starting line. + +Rust, Python, JavaScript/TypeScript, C/C++, Go, Java, C#, Swift, SQL and many +other source formats are supported, including named build files such as +`Makefile` and `Dockerfile`. Add your project folder and let semantic indexing +finish for meaning-based search. See [code search](docs/CODE-SEARCH.md) for +details and agent integration. + ## A little less searching. A little more finding. Matching words stand out in the results and document preview. PDF results diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index ce30583..e5f7274 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -119,6 +119,8 @@ fn set_cpu_threads(threads: usize, state: State<'_, AppState>) -> Result, extension: Option, min_size: Option, @@ -132,6 +134,7 @@ struct Filters { impl From for SearchOptions { fn from(value: Filters) -> Self { Self { + code_only: value.code_only, folder: value.folder, extension: value.extension, min_size: value.min_size, @@ -500,6 +503,53 @@ fn main() { mod tests { use super::*; + #[test] + fn code_scope_defaults_off_and_combines_with_other_filters() { + let legacy: Filters = serde_json::from_str(r#"{"extension":"rs"}"#).unwrap(); + let legacy: SearchOptions = legacy.into(); + assert!(!legacy.code_only); + let scoped: Filters = + serde_json::from_str(r#"{"code_only":true,"extension":"rs","min_size":10}"#).unwrap(); + let scoped: SearchOptions = scoped.into(); + assert!(scoped.code_only); + assert_eq!(scoped.extension.as_deref(), Some("rs")); + assert_eq!(scoped.min_size, Some(10)); + assert!(serde_json::from_str::(r#"{"code_only":"true"}"#).is_err()); + } + + #[test] + fn code_scope_reaches_real_keyword_search_without_model_readiness() { + let root = tempfile::tempdir().unwrap(); + std::fs::write( + root.path().join("parse.rs"), + "fn parse_configuration() { /* validate config */ }", + ) + .unwrap(); + std::fs::write( + root.path().join("notes.md"), + "parse_configuration validate config", + ) + .unwrap(); + let mut index = SearchIndex::open(":memory:").unwrap(); + index.index_folder(root.path()).unwrap(); + let all: SearchOptions = Filters::default().into(); + assert_eq!( + index + .search_with_options("parse_configuration", 10, &all) + .unwrap() + .len(), + 2 + ); + let scoped: SearchOptions = serde_json::from_str::(r#"{"code_only":true}"#) + .unwrap() + .into(); + let hits = index + .search_with_options("parse_configuration", 10, &scoped) + .unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].filename, "parse.rs"); + } + #[test] fn measured_failure_retains_closed_trace_without_inventing_results() { let mut trace = TraceCollector::new(MetricsMode::Debug); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 66504da..b3b93e6 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -12,9 +12,10 @@ import { emptySearchState, LatestLane, semanticAvailable, + scopedFilters, SEMANTIC_SETTLE_MS, } from "./search-state"; -import type { SemanticPhase } from "./search-state"; +import type { SemanticPhase, SearchFilters } from "./search-state"; import { fileIcon, icon } from "./icons"; import { SearchMeasurement, @@ -108,13 +109,13 @@ interface SemanticStatus { interface SearchRequest { id: number; query: string; - filters: Record; + filters: SearchFilters; started: number; measurement: SearchMeasurement; } document.querySelector("#app")!.innerHTML = ` -
searchi
Loading your index…
${icon("filter")}All files${icon("chevron")}

Filter your search

${icon("settings")}

Folders & resources

Search automatically uses supported acceleration with CPU fallback.

    Local timing diagnostics for new searches. Queries and file paths are excluded from copied reports.

    Optional AI answers

    Disabled by default. Only an explicit action sends the shown passage to your configured endpoint.

    Endpoint, model and key stay in this app session. The key authenticates requests to your endpoint.

    +
    searchi
    Loading your index…
    ${icon("filter")}All files${icon("chevron")}

    Filter your search

    ${icon("settings")}

    Folders & resources

    Search automatically uses supported acceleration with CPU fallback.

      Local timing diagnostics for new searches. Queries and file paths are excluded from copied reports.

      Optional AI answers

      Disabled by default. Only an explicit action sends the shown passage to your configured endpoint.

      Endpoint, model and key stay in this app session. The key authenticates requests to your endpoint.

      ${icon("sparkles")}Preparing local semantic search…Local model${icon("chevron")}
      Google EmbeddingGemma 2

      The verified local model downloads automatically on first launch (865 MB). Your files stay on this device.

      @@ -135,6 +136,7 @@ try { } let results: SearchResult[] = []; let displayedLiteralQuery = literalQuery(""); +let displayedCodeScope = false; let explainedSelection = ""; let selected = -1; let requestId = 0; @@ -726,6 +728,7 @@ function select(index: number, focus = false) { : media ? "Preview the matched media below." : "No extracted text is available for this file."); + element("preview-text").classList.toggle("code-passage", displayedCodeScope); renderHighlighted( element("preview-text"), passageText, @@ -833,6 +836,7 @@ function render() { "semantic_video", "semantic_audio", ].includes(result.match_kind); + entry.snippet.classList.toggle("code-snippet", displayedCodeScope); renderHighlighted( entry.snippet, displayExcerpt(result.snippet, displayedLiteralQuery), @@ -879,7 +883,7 @@ function captureFilters() { throw new Error("Enter a valid modification date."); return stamp; }; - return { + return scopedFilters(element("filter-scope").value, { folder: input("filter-folder") || null, extension: input("filter-extension") || null, min_duration_ms: duration("filter-duration-min"), @@ -888,7 +892,7 @@ function captureFilters() { max_size: size("filter-max"), modified_after_unix_ms: date("filter-after", false), modified_before_unix_ms: date("filter-before", true), - }; + }); } function applyResults( @@ -907,6 +911,7 @@ function applyResults( shownLane = lane; displayedRequestId = request.id; displayedLiteralQuery = literalQuery(request.query); + displayedCodeScope = request.filters.code_only === true; results = found; selected = -1; render(); @@ -1209,6 +1214,10 @@ element("browse-folder").addEventListener("click", async () => { } }); element("filters").addEventListener("change", () => { + element("scope-label").textContent = + element("filter-scope").value === "code" + ? "Code" + : "All files"; void search(); }); element("retry-model").addEventListener("click", async () => { diff --git a/apps/desktop/src/match-display.ts b/apps/desktop/src/match-display.ts index 41618ef..5ced54e 100644 --- a/apps/desktop/src/match-display.ts +++ b/apps/desktop/src/match-display.ts @@ -195,7 +195,9 @@ export function matchExplanation(result: MatchResult) { ? ` on page ${source.number}` : source?.kind === "paragraph" ? ` in paragraph ${source.number}` - : ""; + : source?.kind === "line" + ? ` at line ${source.number}` + : ""; const time = source?.kind === "timestamp_ms" ? ` near ${timestampLabel(source.number)}` diff --git a/apps/desktop/src/search-state.ts b/apps/desktop/src/search-state.ts index e1da385..047261c 100644 --- a/apps/desktop/src/search-state.ts +++ b/apps/desktop/src/search-state.ts @@ -1,3 +1,16 @@ +/** Scope is captured with filters; changing controls cannot mutate active work. */ +export type SearchFilters = Readonly< + Record +>; +export function scopedFilters( + scope: string, + filters: Record, +): SearchFilters { + if (scope !== "all" && scope !== "code") + throw new Error("Choose a valid search scope."); + return Object.freeze({ ...filters, code_only: scope === "code" }); +} + /** Let rapid typing settle before starting the model lane. */ export const SEMANTIC_SETTLE_MS = 120; diff --git a/apps/desktop/src/style.css b/apps/desktop/src/style.css index 1cf095f..60e69a1 100644 --- a/apps/desktop/src/style.css +++ b/apps/desktop/src/style.css @@ -937,3 +937,15 @@ mark[data-match="literal"] { border-color: Highlight; } } + +/* Preserve actual source indentation without introducing generated annotations. */ +.snippet.code-snippet, +aside pre.code-passage { + font-family: + ui-monospace, "Cascadia Code", "SFMono-Regular", Consolas, monospace; + white-space: pre-wrap; + tab-size: 4; +} +aside pre.code-passage { + line-height: 1.6; +} diff --git a/apps/desktop/tests/match-display.test.mjs b/apps/desktop/tests/match-display.test.mjs index 80cf5d9..83ff9f4 100644 --- a/apps/desktop/tests/match-display.test.mjs +++ b/apps/desktop/tests/match-display.test.mjs @@ -80,3 +80,8 @@ test('optional reasons use real source routes and never claim semantic word conf assert.equal(matchExplanation({match_kind:'hybrid',source:{kind:'page',number:2}}).semanticPassage,false); assert.doesNotMatch(JSON.stringify(text),/confidence|percent|importance/); }); + +test('code passage reasons retain actual indexed source line', () => { + assert.match(matchExplanation({match_kind:'semantic', source:{kind:'line',number:97}}).reason, /at line 97/); + assert.match(matchExplanation({match_kind:'hybrid', source:{kind:'line',number:97}}).reason, /showing the passage at line 97/); +}); diff --git a/apps/desktop/tests/search-state.test.mjs b/apps/desktop/tests/search-state.test.mjs index 19f3e4a..cca4b21 100644 --- a/apps/desktop/tests/search-state.test.mjs +++ b/apps/desktop/tests/search-state.test.mjs @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { LatestLane, AutomaticIndexGate, emptySearchState, semanticAvailable, SEMANTIC_SETTLE_MS } from '../src/search-state.ts'; +import { LatestLane, AutomaticIndexGate, emptySearchState, semanticAvailable, SEMANTIC_SETTLE_MS, scopedFilters } from '../src/search-state.ts'; const settle = () => new Promise(resolve => setImmediate(resolve)); test('coalesces queued keystrokes and discards stale responses', async () => { @@ -78,3 +78,32 @@ test('old committed semantic generation stays available during rebuild, activati assert.equal(semanticAvailable({ready: true, generation: 2, embedding: false}), true); assert.equal(SEMANTIC_SETTLE_MS, 120); }); + +test('scope snapshots freeze all filters independently of later control changes', () => { + const controls = { extension: 'rs', folder: '/project' }; + const all = scopedFilters('all', controls); + const code = scopedFilters('code', controls); + controls.extension = 'pdf'; + assert.equal(all.code_only, false); + assert.equal(code.code_only, true); + assert.equal(code.extension, 'rs'); + assert.equal(code.folder, '/project'); + assert.throws(() => { code.code_only = false; }, TypeError); + assert.throws(() => scopedFilters('unknown', {}), /scope/); +}); +test('same-query scope change discards old lane output and runs latest scoped request', async () => { + let revision = 1; let release; const seen = []; const accepted = []; + const lane = new LatestLane(async request => { + seen.push(request.filters.code_only); + if (request.id === 1) await new Promise(resolve => { release = resolve; }); + if (request.id === revision) accepted.push(request); + }, request => request.id === revision, () => assert.fail('Unexpected failure')); + lane.submit({id: 1, query: 'parse configuration', filters: scopedFilters('all', {})}); + revision = 2; + lane.submit({id: 2, query: 'parse configuration', filters: scopedFilters('code', {extension: 'rs'})}); + release(); await settle(); + assert.deepEqual(seen, [false, true]); + assert.equal(accepted.length, 1); + assert.equal(accepted[0].filters.code_only, true); + assert.equal(accepted[0].filters.extension, 'rs'); +}); diff --git a/crates/searchi-cli/src/main.rs b/crates/searchi-cli/src/main.rs index 5d79971..25023e7 100644 --- a/crates/searchi-cli/src/main.rs +++ b/crates/searchi-cli/src/main.rs @@ -68,6 +68,9 @@ enum Command { /// Restrict results to this folder and descendants. #[arg(long)] folder: Option, + /// Search indexed source-code files only; supports symbols and semantic code queries. + #[arg(long)] + code_only: bool, /// File extension, with or without the leading dot. #[arg(long)] extension: Option, @@ -166,6 +169,7 @@ fn main() -> Result<()> { performance, debug_performance, folder, + code_only, extension, min_size, max_size, @@ -183,6 +187,7 @@ fn main() -> Result<()> { require_existing_index(&db)?; let options = SearchOptions { folder, + code_only, extension, min_size, max_size, diff --git a/crates/searchi-cli/src/mcp.rs b/crates/searchi-cli/src/mcp.rs index b7a8dfd..d215992 100644 --- a/crates/searchi-cli/src/mcp.rs +++ b/crates/searchi-cli/src/mcp.rs @@ -38,6 +38,9 @@ struct SearchInput { limit: u16, folder: Option, + /// Restrict all retrieval lanes to indexed source-code files. + #[serde(default)] + code_only: bool, extension: Option, @@ -168,6 +171,7 @@ impl AgentServer { let options = SearchOptions { folder: input.folder.map(PathBuf::from), + code_only: input.code_only, extension: input.extension, @@ -272,6 +276,7 @@ impl AgentServer { Ok(json!({ "mode": mode, + "code_only": input.code_only, "semantic_available": mode == "hybrid", diff --git a/crates/searchi-cli/tests/commands.rs b/crates/searchi-cli/tests/commands.rs index 116f061..2453091 100644 --- a/crates/searchi-cli/tests/commands.rs +++ b/crates/searchi-cli/tests/commands.rs @@ -157,3 +157,55 @@ fn exclusions_and_filters_change_real_results() { let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap(); assert_eq!(stats["files"], 2); } + +#[test] +fn code_only_filters_real_results_and_finds_exact_symbols() { + let fixture = Fixture::new(); + fs::create_dir(fixture.0.join("code")).unwrap(); + fs::write( + fixture.0.join("code/retry.rs"), + "pub fn refreshAccessToken() { /* retry failed requests with backoff */ }", + ) + .unwrap(); + fs::write( + fixture.0.join("code/notes.txt"), + "refreshAccessToken retry failed requests with backoff", + ) + .unwrap(); + let indexed = fixture.run(&["index", "--db", "index.db", "--root", "code"]); + assert!( + indexed.status.success(), + "{}", + String::from_utf8_lossy(&indexed.stderr) + ); + let all = fixture.run(&["search", "--db", "index.db", "refreshAccessToken", "--json"]); + assert_eq!( + serde_json::from_slice::(&all.stdout) + .unwrap() + .as_array() + .unwrap() + .len(), + 2 + ); + let code = fixture.run(&[ + "search", + "--db", + "index.db", + "refreshAccessToken", + "--code-only", + "--json", + ]); + assert!( + code.status.success(), + "{}", + String::from_utf8_lossy(&code.stderr) + ); + let results: serde_json::Value = serde_json::from_slice(&code.stdout).unwrap(); + assert_eq!(results.as_array().unwrap().len(), 1); + assert_eq!(results[0]["filename"], "retry.rs"); + assert!(results[0]["snippet"] + .as_str() + .unwrap() + .contains("refreshAccessToken")); + assert_eq!(results[0]["source"]["kind"], "line"); +} diff --git a/crates/searchi-cli/tests/mcp.rs b/crates/searchi-cli/tests/mcp.rs index c1dbfd2..7744c13 100644 --- a/crates/searchi-cli/tests/mcp.rs +++ b/crates/searchi-cli/tests/mcp.rs @@ -75,6 +75,16 @@ fn actual_mcp_handshake_search_and_guarded_sources() { "Termination clause: sixty days written notice to end the supplier partnership.", ) .unwrap(); + std::fs::write( + folder.join("retry.rs"), + "pub fn refreshAccessToken() { /* retry requests */ }", + ) + .unwrap(); + std::fs::write( + folder.join("symbol-note.txt"), + "refreshAccessToken retry requests", + ) + .unwrap(); let db = fixture.path().join("index.sqlite"); let indexed = Command::new(env!("CARGO_BIN_EXE_searchi")) .args(["index", "--db"]) @@ -154,6 +164,21 @@ fn actual_mcp_handshake_search_and_guarded_sources() { .unwrap() .contains("local path")); } + let code = server.request( + 40, + "tools/call", + json!({"name":"search","arguments":{"query":"refreshAccessToken","code_only":true}}), + ); + let code_results = code["result"]["structuredContent"]["results"] + .as_array() + .unwrap(); + assert_eq!(code_results.len(), 1); + assert_eq!(code_results[0]["filename"], "retry.rs"); + assert_eq!(code_results[0]["source"]["kind"], "line"); + assert!(code_results[0]["snippet"] + .as_str() + .unwrap() + .contains("refreshAccessToken")); let mutation = server.request(7, "tools/call", json!({"name":"index","arguments":{}})); assert!(mutation["error"].is_object()); std::fs::write( diff --git a/crates/searchi-core/src/code.rs b/crates/searchi-core/src/code.rs new file mode 100644 index 0000000..4de9afc --- /dev/null +++ b/crates/searchi-core/src/code.rs @@ -0,0 +1,52 @@ +//! Shared code-file classification for extraction and pre-ranking scope filters. +use std::{path::Path, sync::OnceLock}; + +pub const CODE_EXTENSIONS: &[&str] = &[ + "rs", "py", "pyi", "js", "jsx", "mjs", "cjs", "ts", "tsx", "mts", "cts", "c", "h", "cpp", "cc", + "cxx", "hpp", "hxx", "hh", "go", "java", "kt", "kts", "cs", "swift", "m", "mm", "rb", "php", + "phtml", "scala", "sc", "sql", "sh", "bash", "zsh", "fish", "ps1", "psm1", "psd1", "bat", + "cmd", "lua", "r", "pl", "pm", "ex", "exs", "erl", "hrl", "hs", "lhs", "clj", "cljs", "cljc", + "dart", "vue", "svelte", "graphql", "gql", "tf", "tfvars", "proto", "zig", "vb", "vbs", "f", + "for", "f90", "f95", "f03", "pas", "d", "cmake", +]; +const CODE_FILENAMES: &[&str] = &[ + "makefile", + "gnumakefile", + "dockerfile", + "cmakelists.txt", + "justfile", +]; + +pub fn is_code_path(path: &Path) -> bool { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + let filename = path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + CODE_EXTENSIONS.contains(&extension.as_str()) || CODE_FILENAMES.contains(&filename.as_str()) +} + +/// Static trusted SQL predicate for the shared internal `files f` alias. No +/// caller-controlled query, path or extension is interpolated into this SQL. +pub fn code_file_predicate() -> &'static str { + static SQL: OnceLock = OnceLock::new(); + SQL.get_or_init(|| { + let literals = |values: &[&str]| { + values + .iter() + .map(|value| format!("'{value}'")) + .collect::>() + .join(",") + }; + format!( + "(f.extension IN ({}) OR lower(f.filename) IN ({}))", + literals(CODE_EXTENSIONS), + literals(CODE_FILENAMES) + ) + }) +} diff --git a/crates/searchi-core/src/extraction.rs b/crates/searchi-core/src/extraction.rs index cdf2ed2..167afb1 100644 --- a/crates/searchi-core/src/extraction.rs +++ b/crates/searchi-core/src/extraction.rs @@ -33,27 +33,30 @@ pub(crate) fn extract(path: &Path) -> Result>> { docx(&bytes).map(Some) }; } - if !matches!( - extension.as_str(), - "txt" - | "md" - | "rs" - | "py" - | "js" - | "ts" - | "tsx" - | "jsx" - | "json" - | "toml" - | "yaml" - | "yml" - | "csv" - | "html" - | "css" - | "xml" - | "log" - | "sh" - ) { + let code = crate::is_code_path(path); + if !code + && !matches!( + extension.as_str(), + "txt" + | "md" + | "rs" + | "py" + | "js" + | "ts" + | "tsx" + | "jsx" + | "json" + | "toml" + | "yaml" + | "yml" + | "csv" + | "html" + | "css" + | "xml" + | "log" + | "sh" + ) + { return Ok(None); } let file = File::open(path)?; @@ -69,6 +72,26 @@ pub(crate) fn extract(path: &Path) -> Result>> { return Ok(None); }; let lines = text.split_inclusive('\n').collect::>(); + if code { + let mut passages = Vec::new(); + for start in (0..lines.len()).step_by(96) { + if passages.len() >= MAX_PASSAGES { + return Ok(None); + } + let end = (start + 128).min(lines.len()); + passages.push(Passage { + text: lines[start..end].concat(), + source: SourceLocation { + kind: "line".into(), + number: (start + 1) as u32, + }, + }); + if end == lines.len() { + break; + } + } + return Ok(Some(passages)); + } Ok(Some( lines .chunks(64) diff --git a/crates/searchi-core/src/lib.rs b/crates/searchi-core/src/lib.rs index 18cd119..a1b4c09 100644 --- a/crates/searchi-core/src/lib.rs +++ b/crates/searchi-core/src/lib.rs @@ -3,7 +3,9 @@ //! Reconciliation is atomic. Incomplete directory walks never delete records. //! Text and document extraction have byte limits, but still run in this process. //! Worker isolation remains a production milestone. +mod code; mod extraction; +pub use code::{code_file_predicate, is_code_path, CODE_EXTENSIONS}; pub mod metrics; use anyhow::{bail, Context, Result}; pub use metrics::{MetricsMode, SearchMetrics, SearchResponse, SearchSpan}; @@ -19,7 +21,7 @@ use std::{ }; use walkdir::WalkDir; -const SCHEMA_VERSION: i64 = 10; +const SCHEMA_VERSION: i64 = 11; const MAX_RESULTS: usize = 1000; #[derive(Debug, Default, Serialize)] @@ -50,6 +52,8 @@ impl Default for IndexOptions { /// The folder is canonicalized and includes descendants, with a path-boundary check. #[derive(Debug, Default, Clone)] pub struct SearchOptions { + /// Restrict every retrieval lane to source/build-script files before top-K. + pub code_only: bool, pub folder: Option, pub extension: Option, pub min_size: Option, @@ -242,6 +246,16 @@ impl SearchIndex { CREATE TRIGGER IF NOT EXISTS files_media_delete AFTER DELETE ON files BEGIN DELETE FROM media_metadata WHERE path=old.path; END; CREATE TRIGGER IF NOT EXISTS files_media_update AFTER UPDATE ON files WHEN new.path!=old.path OR new.size!=old.size OR new.mtime!=old.mtime BEGIN DELETE FROM media_metadata WHERE path=old.path; END; PRAGMA user_version=10;")?; + if version < 11 { + initialization.execute( + &format!( + "UPDATE files AS f SET mtime=-1 WHERE {}", + code_file_predicate() + ), + [], + )?; + } + initialization.execute_batch("PRAGMA user_version=11;")?; initialization.commit()?; } Ok(Self { @@ -620,12 +634,16 @@ impl SearchIndex { trace.end(normalization, Default::default()); let planning = trace.begin("lexical.candidate_plan", parent); let limit = limit.min(MAX_RESULTS); - let filters = " AND (?3 IS NULL OR substr(f.path,1,length(?3))=?3) + let mut filters = String::from(" AND (?3 IS NULL OR substr(f.path,1,length(?3))=?3) AND (?4 IS NULL OR f.extension=?4) AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6) AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8) AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10)) - AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))"; + AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))"); + if options.code_only { + filters.push_str(" AND "); + filters.push_str(code_file_predicate()); + } let filter_values = [ folder_prefix.map_or(Value::Null, Value::Text), extension.map_or(Value::Null, Value::Text), @@ -639,6 +657,7 @@ impl SearchIndex { .map_or(Value::Null, Value::Integer), ]; let has_filters = filter_values.iter().any(|value| *value != Value::Null) + || options.code_only || options.min_duration_ms.is_some() || options.max_duration_ms.is_some(); let mut results = Vec::new(); diff --git a/crates/searchi-core/tests/code_search.rs b/crates/searchi-core/tests/code_search.rs new file mode 100644 index 0000000..3e6f606 --- /dev/null +++ b/crates/searchi-core/tests/code_search.rs @@ -0,0 +1,92 @@ +use anyhow::Result; +use searchi_core::{SearchIndex, SearchOptions, SourceLocation}; +use std::fs; + +#[test] +fn code_scope_filters_before_limit_and_extracts_build_files() -> Result<()> { + let root = tempfile::tempdir()?; + for number in 0..100 { + fs::write( + root.path().join(format!("early{number}.txt")), + "refreshAccessToken", + )?; + } + fs::write( + root.path().join("last.go"), + "func refreshAccessToken() {}\n", + )?; + fs::write( + root.path().join("Makefile"), + "refreshAccessToken:\n\techo ready\n", + )?; + fs::write(root.path().join("large.rs"), "\n".repeat(1_000_000))?; + let mut index = SearchIndex::open(":memory:")?; + index.index_folder(root.path())?; + assert_eq!(index.search("large.rs", 1)?[0].filename, "large.rs"); + let options = SearchOptions { + code_only: true, + ..Default::default() + }; + let results = index.search_with_options("refreshAccessToken", 2, &options)?; + assert_eq!(results.len(), 2); + assert!(results + .iter() + .all(|r| r.filename == "last.go" || r.filename == "Makefile")); + assert!(results + .iter() + .all(|r| r.source.is_some() && r.snippet.contains("refreshAccessToken"))); + Ok(()) +} + +#[test] +fn overlapping_source_passages_preserve_routine_and_actual_lines() -> Result<()> { + let root = tempfile::tempdir()?; + let file = root.path().join("module.swift"); + let mut lines = vec!["// filler\n"; 166]; + lines[116] = "func importantRoutine() {\n"; + lines[136] = " return finalValue\n"; + lines[137] = "}\n"; + fs::write(&file, lines.concat())?; + let mut index = SearchIndex::open(":memory:")?; + index.index_folder(root.path())?; + let path = fs::canonicalize(file)?.to_string_lossy().into_owned(); + let passage = index + .source_passage( + &path, + &SourceLocation { + kind: "line".into(), + number: 97, + }, + 8192, + )? + .unwrap(); + assert_eq!(passage.text, lines[96..].concat()); + assert!(passage.text.contains("importantRoutine") && passage.text.contains("finalValue")); + Ok(()) +} + +#[test] +fn old_code_index_reextracts_unchanged_source_on_upgrade() -> Result<()> { + let root = tempfile::tempdir()?; + let files = root.path().join("files"); + fs::create_dir(&files)?; + fs::write(files.join("module.java"), "class Evidence {}")?; + fs::write(files.join("notes.txt"), "ordinary evidence")?; + let db = root.path().join("index.db"); + let mut index = SearchIndex::open(&db)?; + index.index_folder(&files)?; + drop(index); + let connection = rusqlite::Connection::open(&db)?; + connection.execute_batch("PRAGMA user_version=10;")?; + drop(connection); + let mut index = SearchIndex::open(&db)?; + assert_eq!(index.search("module.java", 1)?[0].filename, "module.java"); + let report = index.index_folder(&files)?; + assert_eq!(report.indexed, 1); + assert_eq!(report.unchanged, 1); + assert!(index + .search("Evidence", 10)? + .iter() + .any(|r| r.filename == "module.java" && r.source.is_some())); + Ok(()) +} diff --git a/crates/searchi-semantic/native/worker.cpp b/crates/searchi-semantic/native/worker.cpp index 5e8febc..fb55150 100644 --- a/crates/searchi-semantic/native/worker.cpp +++ b/crates/searchi-semantic/native/worker.cpp @@ -197,11 +197,50 @@ public: require(!text.empty() && text.size() <= max_passage_bytes, "Passage exceeds 1 MiB or is empty"); auto tokens = tokenize(text,false); require(!tokens.empty(), "Passage has no tokens"); - // Preserve actual source offsets, rather than inventing locations from - // a lossy decoded representation. Unexpected normalization is an error. - require(decode(tokens,tokens.size()) == text, "Tokenizer roundtrip changed source text; cannot verify source offsets"); size_t prompt = tokenize(document_prefix(title),true).size(); require(prompt + 132 < size_t(context_tokens), "Document prefix exceeds token budget"); + // SentencePiece may normalize source characters. In that case decoded + // token lengths are not source offsets: split the original UTF-8 bytes + // and verify each complete prefixed fragment against the real tokenizer. + if (decode(tokens,tokens.size()) != text) { + const auto prefix = document_prefix(title); + json result = json::array(); + if (prefix.size()+text.size() <= max_text_bytes && tokenize(prefix+text,true).size() <= size_t(context_tokens)) { + return json::array({{{"text",text},{"start_byte",0},{"end_byte",text.size()}}}); + } + std::vector boundaries; + for (size_t i=0;i start, "Cannot fit source fragment in model context"); + auto fragment = text.substr(start,best-start); + require(tokenize(prefix+fragment,true).size() <= size_t(context_tokens), "Source fragment exceeds context budget"); + result.push_back({{"text",fragment},{"start_byte",start},{"end_byte",best}}); + if (best == text.size()) break; + // Bounded original-byte overlap; always advance, even for + // unusual Unicode/token expansion. No decoded text is indexed. + size_t next = best - std::min(size_t(128),(best-start)/2); + start_index = size_t(std::lower_bound(boundaries.begin(),boundaries.end(),next)-boundaries.begin()); + } + return result; + } size_t budget = size_t(context_tokens) - prompt - 4; size_t stride = budget - 128; json result = json::array(); @@ -253,10 +292,11 @@ public: return {{"vectors",json::array({embed_bitmap(bitmap.get())})}}; } if (op == "windows") return {{"windows",windows(request.at("text").get(),request.value("title",std::string()))}}; - if (op == "query") { + if (op == "query" || op == "code_query") { bool debug = request.value("debug_performance",false); json timings; - auto vector = embed("task: search result | query: "+request.at("text").get(),512,debug ? &timings : nullptr); + auto prefix = op == "code_query" ? "task: code retrieval | query: " : "task: search result | query: "; + auto vector = embed(prefix+request.at("text").get(),512,debug ? &timings : nullptr); json response = {{"vectors",json::array({vector})}}; if (debug) response["performance"] = timings; return response; @@ -289,7 +329,7 @@ int main(int argc, char ** argv) { bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1"; Encoder encoder(path,threads,device == "auto" && !force_cpu); std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions}, - {"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true}}).dump() << '\n' << std::flush; + {"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true},{"code_query_supported",true}}).dump() << '\n' << std::flush; for (;;) { std::string line; char c; while (std::cin.get(c) && c != '\n') { diff --git a/crates/searchi-semantic/src/gemma.rs b/crates/searchi-semantic/src/gemma.rs index 72b59ae..4a3fc06 100644 --- a/crates/searchi-semantic/src/gemma.rs +++ b/crates/searchi-semantic/src/gemma.rs @@ -48,6 +48,13 @@ pub struct WorkerInfo { pub runtime: String, #[serde(default)] pub media_supported: bool, + #[serde(default)] + pub code_query_supported: bool, +} + +fn query_operation(info: &WorkerInfo, code: bool) -> Result<&'static str> { + ensure!(!code || info.code_query_supported,"Installed embedding worker does not support code retrieval; repair/rebuild the native runtime"); + Ok(if code { "code_query" } else { "query" }) } /// Persistent isolated native worker. No HTTP or remote model fallback. @@ -317,15 +324,24 @@ impl GemmaEmbedder { pub fn embed_query(&mut self, query: &str) -> Result> { self.query_profile = None; - self.embed_query_inner(query, false) + self.embed_query_inner(query, false, false) } /// Opt-in diagnostics contain only durations and counts, never query text. pub fn embed_query_debug(&mut self, query: &str) -> Result> { self.query_profile = Some(BTreeMap::new()); - self.embed_query_inner(query, true) + self.embed_query_inner(query, true, false) } + /// Google code retrieval task; unavailable workers fail explicitly. + pub fn embed_code_query(&mut self, query: &str) -> Result> { + self.query_profile = None; + self.embed_query_inner(query, false, true) + } + pub fn embed_code_query_debug(&mut self, query: &str) -> Result> { + self.query_profile = Some(BTreeMap::new()); + self.embed_query_inner(query, true, true) + } pub fn query_diagnostics(&self) -> BTreeMap { self.query_profile.clone().unwrap_or_default() } @@ -337,13 +353,13 @@ impl GemmaEmbedder { } } - fn embed_query_inner(&mut self, query: &str, debug: bool) -> Result> { + fn embed_query_inner(&mut self, query: &str, debug: bool, code: bool) -> Result> { ensure!( !query.trim().is_empty() && query.len() <= 4096, "Query must be nonempty and at most 4096 UTF-8 bytes" ); - let response = - self.request(json!({"op":"query","text":query,"debug_performance":debug}))?; + let op = query_operation(&self.info, code)?; + let response = self.request(json!({"op":op,"text":query,"debug_performance":debug}))?; if debug { if let Some(profile) = &mut self.query_profile { for name in [ @@ -655,6 +671,12 @@ mod tests { let mut handshake = json!({"ready":true,"model":"google/embeddinggemma-2","dimensions":768,"device":"cpu","threads":1,"runtime":"b11461-no-logits-v1"}); let old: WorkerInfo = serde_json::from_value(handshake.clone()).unwrap(); assert!(!old.media_supported); + assert!(!old.code_query_supported); + assert!(query_operation(&old, true).is_err()); + assert_eq!(query_operation(&old, false).unwrap(), "query"); + let mut new = old.clone(); + new.code_query_supported = true; + assert_eq!(query_operation(&new, true).unwrap(), "code_query"); handshake["media_supported"] = json!(true); assert!( serde_json::from_value::(handshake) diff --git a/crates/searchi-semantic/tests/code_cpu.rs b/crates/searchi-semantic/tests/code_cpu.rs new file mode 100644 index 0000000..2219c8a --- /dev/null +++ b/crates/searchi-semantic/tests/code_cpu.rs @@ -0,0 +1,66 @@ +use anyhow::Result; +use searchi_semantic::{Device, GemmaEmbedder}; + +#[test] +#[ignore = "Requires explicitly provisioned CPU Google tokenizer and worker"] +fn normalized_code_tokenization_preserves_original_source_windows() -> Result<()> { + let models = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set model directory"); + let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set CPU worker"); + let mut embedder = GemmaEmbedder::load_with_device(models, worker, 1, Device::Cpu)?; + // Genuine frozen TypeScript fixture that does not roundtrip through the + // tokenizer. Do not replace source bytes with normalized decoded tokens. + let code = "export function process(rows: {category: string; amount: number}[]) {\n const totals = new Map();\n for (const row of rows) totals.set(row.category, (totals.get(row.category) ?? 0) + row.amount);\n return totals;\n}\n"; + for text in [ + code.to_owned(), + format!("{code}// Grüße 日本語 🦀\n").repeat(160), + ] { + let windows = embedder.split_windows(&text, "c.ts")?; + let mut covered = 0; + for window in &windows { + assert!(window.start_byte <= covered); + assert!(window.end_byte > covered); + assert_eq!(window.text, text[window.start_byte..window.end_byte]); + covered = window.end_byte; + } + assert_eq!(covered, text.len()); + assert!(windows.len() <= 256); + if text.len() > 1000 { + assert!(windows.len() > 1); + } + } + Ok(()) +} + +#[test] +#[ignore = "Requires explicitly provisioned CPU Google model and rebuilt code-capable worker"] +fn actual_code_task_differs_from_search_and_debug_preserves_vectors() -> Result<()> { + let models = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set model directory"); + let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set CPU worker"); + let mut embedder = GemmaEmbedder::load_with_device(models, worker, 1, Device::Cpu)?; + assert!(embedder.info.code_query_supported); + assert_eq!(embedder.info.device, "cpu"); + let query = "parse a JSON response and retry a failed network request"; + let ordinary = embedder.embed_query(query)?; + let code = embedder.embed_code_query(query)?; + let cosine = |a: &[f32], b: &[f32]| { + a.iter() + .zip(b) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum::() + }; + assert!( + cosine(&ordinary, &code) < 0.999999, + "Task-specific prompt must not silently reuse generic search embeddings" + ); + let debug = embedder.embed_code_query_debug(query)?; + assert!(cosine(&code, &debug) > 0.999999); + assert!(embedder + .query_diagnostics() + .contains_key("worker_token_count")); + let ordinary_again = embedder.embed_query(query)?; + assert!( + cosine(&ordinary, &ordinary_again) > 0.999999, + "Previous code query must not contaminate a retained encoder context" + ); + Ok(()) +} diff --git a/crates/searchi-service/src/lib.rs b/crates/searchi-service/src/lib.rs index 2e37f9f..6c01200 100644 --- a/crates/searchi-service/src/lib.rs +++ b/crates/searchi-service/src/lib.rs @@ -54,6 +54,12 @@ pub trait EmbeddingProvider: Send { fn split_windows(&mut self, text: &str, title: &str) -> Result>; fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result>>; fn embed_query(&mut self, query: &str) -> Result>; + fn embed_code_query(&mut self, _query: &str) -> Result> { + anyhow::bail!("Provider does not support Google code retrieval") + } + fn embed_code_query_debug(&mut self, query: &str) -> Result> { + self.embed_code_query(query) + } fn embed_query_debug(&mut self, query: &str) -> Result> { self.embed_query(query) } @@ -90,6 +96,12 @@ impl EmbeddingProvider for GemmaEmbedder { fn embed_query_debug(&mut self, query: &str) -> Result> { GemmaEmbedder::embed_query_debug(self, query) } + fn embed_code_query(&mut self, query: &str) -> Result> { + GemmaEmbedder::embed_code_query(self, query) + } + fn embed_code_query_debug(&mut self, query: &str) -> Result> { + GemmaEmbedder::embed_code_query_debug(self, query) + } fn query_diagnostics(&self) -> std::collections::BTreeMap { GemmaEmbedder::query_diagnostics(self) } @@ -132,6 +144,11 @@ pub struct EmbeddingStatus { pub pending: usize, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum QueryTask { + Search, + Code, +} /// Shared local hybrid service. Google EmbeddingGemma 2 is the default and /// only shipped provider; no alternative model is silently substituted. pub struct HybridSearch { @@ -142,7 +159,7 @@ pub struct HybridSearch { // Real provider outputs only, scoped to this model session. Queries never // persist and results/filters are never cached here. query_cache_fingerprint: String, - query_cache: VecDeque<(String, Vec)>, + query_cache: VecDeque<(QueryTask, String, Vec)>, } impl HybridSearch { @@ -396,7 +413,7 @@ impl HybridSearch { trace.timed("hybrid.identifier_priority", parent, |_, _| { promote_identifiers(query.trim(), &mut lexical) }); - let vector = self.query_vector(query, trace, parent)?; + let vector = self.query_vector(query, options.code_only, trace, parent)?; let semantic = self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?; let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| { @@ -419,13 +436,14 @@ impl HybridSearch { return Ok(Vec::new()); } let mut trace = TraceCollector::new(MetricsMode::Basic); - let vector = self.query_vector(query, &mut trace, None)?; + let vector = self.query_vector(query, options.code_only, &mut trace, None)?; self.routed_semantic(query, &vector, limit.min(1000), options) } fn query_vector( &mut self, query: &str, + code: bool, trace: &mut TraceCollector, parent: Option, ) -> Result> { @@ -435,6 +453,12 @@ impl HybridSearch { "Query must be nonempty and at most 4096 UTF-8 bytes" ); let span = trace.begin("semantic.query_embedding", parent); + let task = if code { + QueryTask::Code + } else { + QueryTask::Search + }; + trace.counter(span, "code_task", u64::from(code)); if self.query_cache_fingerprint != self.embedder.fingerprint() { self.query_cache.clear(); self.query_cache_fingerprint = self.embedder.fingerprint().to_owned(); @@ -442,13 +466,13 @@ impl HybridSearch { if let Some(position) = self .query_cache .iter() - .position(|(cached, _)| cached == query) + .position(|(cached_task, cached, _)| *cached_task == task && cached == query) { let entry = self .query_cache .remove(position) .expect("Located cache entry"); - let vector = entry.1.clone(); + let vector = entry.2.clone(); self.query_cache.push_back(entry); trace.counter(span, "query_cache_hit", 1); trace.counter(span, "query_cache_miss", 0); @@ -460,10 +484,11 @@ impl HybridSearch { trace.counter(span, "query_cache_hit", 0); trace.counter(span, "query_cache_miss", 1); let vector = (|| { - let vector = if trace.is_debug() { - self.embedder.embed_query_debug(query)? - } else { - self.embedder.embed_query(query)? + let vector = match (code, trace.is_debug()) { + (true, true) => self.embedder.embed_code_query_debug(query)?, + (true, false) => self.embedder.embed_code_query(query)?, + (false, true) => self.embedder.embed_query_debug(query)?, + (false, false) => self.embedder.embed_query(query)?, }; ensure!( vector.len() == self.embedder.dimensions() @@ -487,7 +512,7 @@ impl HybridSearch { self.query_cache.pop_front(); } self.query_cache - .push_back((query.to_owned(), vector.clone())); + .push_back((task, query.to_owned(), vector.clone())); Ok(vector) } } diff --git a/crates/searchi-service/src/routing.rs b/crates/searchi-service/src/routing.rs index a2a96ff..2173b5c 100644 --- a/crates/searchi-service/src/routing.rs +++ b/crates/searchi-service/src/routing.rs @@ -116,7 +116,11 @@ impl HybridSearch { parent: Option, ) -> Result> { trace.timed("semantic.routing", parent, |trace, parent| { - if let Some(origin) = preferred_origin(query) { + if let Some(origin) = if options.code_only { + None + } else { + preferred_origin(query) + } { trace.counter(parent, "audio_preference", 1); let preferred = self.store.search_origin_traced( vector, @@ -157,7 +161,8 @@ impl HybridSearch { if query.trim().is_empty() || limit == 0 { return Ok(Vec::new()); } - let vector = self.embedder.embed_query(query.trim())?; + let mut trace = TraceCollector::new(MetricsMode::Basic); + let vector = self.query_vector(query, options.code_only, &mut trace, None)?; self.store.search_origin( &vector, &self.generation_fingerprint, diff --git a/crates/searchi-service/src/store.rs b/crates/searchi-service/src/store.rs index 016d839..1abe5a3 100644 --- a/crates/searchi-service/src/store.rs +++ b/crates/searchi-service/src/store.rs @@ -28,6 +28,7 @@ struct FilterKey { min_duration: Option, max_duration: Option, source_type: Option, + code_only: bool, } struct EligibilityCache { @@ -551,6 +552,7 @@ impl Store { min_duration: options.min_duration_ms, max_duration: options.max_duration_ms, source_type: source_type.map(str::to_owned), + code_only: options.code_only, }; trace.end(filter_span, Default::default()); let eligibility_span = trace.begin("semantic.eligibility", parent); @@ -906,7 +908,12 @@ fn collect_eligibility( filter: &FilterKey, path: Option<&str>, ) -> Result { - let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c + let code = if filter.code_only { + format!(" AND {}", searchi_core::code_file_predicate()) + } else { + String::new() + }; + let mut statement=connection.prepare(&format!("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime)) AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3) @@ -915,7 +922,7 @@ fn collect_eligibility( AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10)) AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11)) AND (?12 IS NULL OR c.source_type=?12) - AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?; + AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?; let mut rows = statement.query(params![ generation, filter.folder, @@ -971,6 +978,78 @@ mod tests { use searchi_core::SearchIndex; const DIMENSIONS: usize = 768; + #[test] + fn code_filter_precedes_ann_top_k_and_cache_distinguishes_scope() -> Result<()> { + // Retrieval geometry only: production vectors come from Google. + let directory = tempfile::tempdir()?; + let root = directory.path().join("sources"); + std::fs::create_dir(&root)?; + for n in 0..40 { + std::fs::write( + root.join(format!("document-{n}.md")), + "parse JSON and retry requests", + )?; + } + std::fs::write( + root.join("network.rs"), + "fn retry_request() { parse_json(); }", + )?; + std::fs::write(root.join("Makefile"), "build: compile_sources\n")?; + let db = directory.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let mut store = Store::open(&db, DIMENSIONS)?; + let generation = store.begin("code-filter-proof")?; + let mut query = vec![0.; DIMENSIONS]; + query[0] = 1.; + let mut weaker = query.clone(); + weaker[0] = 0.9; + weaker[1] = (1.0_f32 - 0.9_f32.powi(2)).sqrt(); + for passage in lexical.passage_batch(0, 100)? { + let vector = if searchi_core::is_code_path(Path::new(&passage.path)) { + weaker.clone() + } else { + query.clone() + }; + let window = TextWindow { + text: passage.text.clone(), + start_byte: 0, + end_byte: passage.text.len(), + }; + store.insert(generation, &passage, 0, &[window], &[vector])?; + } + store.activate(generation)?; + assert!( + store.search(&query, "code-filter-proof", 1, &SearchOptions::default())?[0] + .filename + .ends_with(".md") + ); + let code = SearchOptions { + code_only: true, + ..Default::default() + }; + let hits = store.search(&query, "code-filter-proof", 2, &code)?; + assert_eq!(hits.len(), 2); + assert!(hits + .iter() + .all(|h| searchi_core::is_code_path(Path::new(&h.path)))); + let incompatible = SearchOptions { + code_only: true, + extension: Some("md".into()), + ..Default::default() + }; + assert!(store + .search(&query, "code-filter-proof", 2, &incompatible)? + .is_empty()); + std::fs::remove_file(root.join("network.rs"))?; + lexical.index_folder(&root)?; + assert_eq!( + store.search(&query, "code-filter-proof", 2, &code)?.len(), + 1 + ); + Ok(()) + } + #[test] fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> { // Geometry verifies grouped retrieval, not model inference or codecs. diff --git a/crates/searchi-service/tests/query_cache.rs b/crates/searchi-service/tests/query_cache.rs index e2cb166..221e39d 100644 --- a/crates/searchi-service/tests/query_cache.rs +++ b/crates/searchi-service/tests/query_cache.rs @@ -48,6 +48,9 @@ impl EmbeddingProvider for CountingProvider { } Ok(vector) } + fn embed_code_query(&mut self, query: &str) -> Result> { + self.embed_query(&format!("code-task:{query}")) + } fn query_diagnostics(&self) -> BTreeMap { BTreeMap::from([("worker_inference_us".into(), 123)]) } @@ -63,6 +66,37 @@ fn count(calls: &Calls) -> usize { calls.queries.lock().unwrap().len() } +#[test] +fn query_task_scopes_cache_and_code_debug_never_replays_old_worker_counters() -> Result<()> { + let (_directory, mut service, calls) = fixture()?; + let normal = SearchOptions::default(); + let code = SearchOptions { + code_only: true, + ..Default::default() + }; + service.search("parse JSON", 5, &normal)?; + service.search("parse JSON", 5, &code)?; + assert_eq!( + *calls.queries.lock().unwrap(), + vec!["parse JSON", "code-task:parse JSON"] + ); + let cached = service.search_measured("parse JSON", 5, &code, true)?; + assert_eq!(count(&calls), 2); + let embedding = cached + .metrics + .trace + .unwrap() + .into_iter() + .find(|s| s.name == "semantic.query_embedding") + .unwrap(); + assert_eq!(embedding.counters["code_task"], 1); + assert_eq!(embedding.counters["query_cache_hit"], 1); + assert!(!embedding.counters.contains_key("worker_inference_us")); + service.search("parse JSON", 5, &normal)?; + assert_eq!(count(&calls), 2); + Ok(()) +} + #[test] fn exact_trimmed_queries_reuse_vectors_but_case_and_model_fingerprint_do_not() -> Result<()> { let (_directory, mut service, calls) = fixture()?; diff --git a/docs/CODE-QUALITY.md b/docs/CODE-QUALITY.md new file mode 100644 index 0000000..77e70e3 --- /dev/null +++ b/docs/CODE-QUALITY.md @@ -0,0 +1,78 @@ +# Source-code relevance checkpoint + +Recorded 2026-10-12 using the actual verified Google EmbeddingGemma 2 Q8_0 +model, rebuilt native worker, shared hybrid service and MCP protocol. Windows +VM, CPU only, one inference thread. No simulated provider or injected vectors. + +Google describes local code retrieval in its +[launch blog](https://blog.google/innovation-and-ai/technology/developers-tools/embeddinggemma-2/). +Its [model card](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2) +specifies `task: code retrieval | query: {query}` for queries and +`title: {filename} | text: {code}` for indexed source. searchi uses that pairing +in Code scope. Google's published benchmark measures their full-precision +checkpoint; it is not a searchi benchmark. + +## Frozen small regression + +[The evaluator](../tools/code-eval/evaluate.py) fixes the implementations, +queries and expected files before inference. Eight short source files use +generic filenames; a ninth ordinary document deliberately contains search +terms. All nine files are indexed, so code filtering must exclude that document +before ranking. The expected code excerpts are checked byte-for-byte through +MCP `read_source`, including line locations. Fixture hashes and every returned +rank are recorded in [raw CPU evidence](evidence/code-search-cpu.json). + +| Query intent | Source | Lexical rank | Hybrid rank | +| --- | --- | --- | --- | +| Streaming file SHA256 | Python | absent | 1 | +| Lower bound in sorted numbers | C++ | absent | 1 | +| Sum amounts by category | TypeScript | absent | 1 | +| Retry requests with exponential backoff | Python | absent | 1 | +| Release semaphore after async failure | C# | absent | 1 | +| Escape unsafe HTML characters | PHP | absent | 1 | +| Ten most expensive records | Python | absent | 1 | +| File verification without full memory load, German | Python | absent | 1 | +| Exact `refreshAccessToken` symbol | Rust | 1 | 1 | + +Top-1 success: **9/9 hybrid, 1/9 lexical**. Every expected hybrid source was +verified and no ordinary document leaked into Code scope. Measured MCP +roundtrips for these nine uncached code queries ranged from **63.88–228.21 ms**, +median **110.76 ms**. Those timings exclude process/model startup and describe +this tiny corpus only; they are not desktop first-use or large-project targets. + +## Defect discovered and fixed + +The first actual indexing attempt failed on TypeScript: Google's tokenizer +normalized source characters, invalidating a detokenization-based byte-offset +assumption. The worker now retains original UTF-8 source, checks the complete +prefixed token budget, and splits normalized long inputs on original character +boundaries. It returns exact original slices with bounded overlap and complete +tail coverage. Short code that fits uses a direct fast path. A separate actual +CPU regression covers the failing TypeScript plus roughly 38 KB of repeated +code with German, Japanese and emoji. Fixtures were not edited to hide failure. + +Additional checks cover strict filtering before lexical/ANN candidate limits, +named build files, overlapping routine context, schema upgrades, task-specific +query caching, actual code/general vector separation and debug-vector parity. + +## Reproduce and limits + +Build the CLI and provision the pinned CPU worker/model as documented in +[BUILDING.md](BUILDING.md) and [SEMANTIC.md](SEMANTIC.md), then run: + +```powershell +$env:SEARCHI_FORCE_CPU = '1' +python tools/code-eval/evaluate.py +``` + +The script uses the repository's generated `.searchi/code-eval` folder and +default local runtime/model paths. It does not index private project folders. +Outputs stay in that generated folder until deliberately reviewed and copied +into documentation evidence. + +These are synthetic, short implementations with one intended answer per query. +They do not prove general project accuracy, declaration-first ranking, +call-graph understanding or reliable no-answer detection. Extraction is +line-based, not an AST parser. Independent real repositories, difficult +near-duplicates, more languages, native UI interaction, other operating systems +and actual GPU execution remain open evaluation gates. diff --git a/docs/CODE-SEARCH.md b/docs/CODE-SEARCH.md new file mode 100644 index 0000000..1683bd3 --- /dev/null +++ b/docs/CODE-SEARCH.md @@ -0,0 +1,80 @@ +# Search source code + +searchi searches source files already included in your selected indexed folders. +In the desktop, open **All files** and select **Code** in the scope selector. +Search an exact symbol or describe the implementation you want. The preview +preserves whitespace and reports the passage's starting source line. +Use `--code-only` in the CLI, or `code_only: true` in MCP, to restrict every +retrieval lane to source-code files before candidate ranking. This is a file-type +filter, not an access-control boundary or an instruction to index new folders. +Other folder, extension, size and date filters still apply. + +Source support includes Rust, Python, JS/TS, C/C++, Go, Java/Kotlin, C#, Swift, +Ruby, PHP, SQL, shells, PowerShell and additional common languages. Named +`Makefile`, `GNUmakefile`, `Dockerfile`, `CMakeLists.txt` and `Justfile` are +included. Ordinary JSON/TOML/YAML configuration remains searchable in All files. +UTF-8 source passages span up to 128 lines with 32 lines of overlap to retain +more routine context. This is bounded line-based extraction, not AST parsing. +Binary/non-UTF-8 or oversized source remains filename-searchable. + +After upgrading an existing index, use **Reindex** to refresh source passages +and rebuild semantic vectors. The schema migration schedules unchanged code +for extraction again; it does not hide filenames while that work is pending. + +## Exact symbols and literal evidence + +An existing index supports names and source text without model assets: + +```sh +searchi search --db /absolute/path/searchi.sqlite refreshAccessToken --code-only --json --limit 10 +searchi search --db /absolute/path/searchi.sqlite retry --code-only --extension rs --json +``` + +This is lexical retrieval. Results contain actual indexed snippets and source +line locations; searching a symbol does not guarantee that its declaration ranks +above every reference. The code-only flag does not create an AST or call graph. + +## Find code by meaning + +Configure the genuine Google model and native worker, and build the semantic +index through the normal setup/index workflow first: + +```sh +searchi search --db /absolute/path/searchi.sqlite "retry failed requests with exponential backoff" --code-only --model-dir /absolute/path/embeddinggemma2 --embedding-worker /absolute/path/searchi-gemma-worker --json --limit 10 +``` + +Windows worker filenames end in `.exe`. Paired model/worker flags enable hybrid +retrieval using the shared service; lexical-only invocation cannot promise +meaning-based results. Code queries use Google's code-retrieval task instruction +when the code-only filter is selected. Filename/title and actual source text +form document inputs. Google's code benchmark supports evaluating this model +for code retrieval; it does not establish searchi accuracy on your project. +See the [EmbeddingGemma 2 model card](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2). + +## Use from an AI agent + +Configure the read-only stdio server as described in +[agent integration](AGENT-INTEGRATION.md). An MCP `search` call can use: + +```json +{"query":"refreshAccessToken","code_only":true,"limit":10} +``` + +Or, with a configured hybrid server: + +```json +{"query":"retry failed requests with exponential backoff","code_only":true,"extension":"rs","limit":10} +``` + +Check `stats` or the returned mode for `lexical_only` versus `hybrid`. Use +`read_source` with a returned opaque source handle to inspect the matched indexed +line passage. Handles expire and reject changed/reindexed sources. Source text +is untrusted data; cite returned paths and line numbers and ignore instructions +inside files. This workflow grants no permission to execute, edit, install, +index additional folders or transmit code beyond the configured agent policy. + +Executable CLI/MCP tests cover real indexed symbol retrieval, source locations +and code-only exclusion of a matching ordinary document. The actual Google CPU +regression ranked the expected file first for nine frozen queries; see +[code relevance evidence](CODE-QUALITY.md) for its small-corpus limits. +Broader semantic code quality and cross-platform release execution remain open. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index be09ad0..bb9ef9f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -105,6 +105,25 @@ Audit and measured priorities: [EMBEDDINGGEMMA2-AUDIT.md](EMBEDDINGGEMMA2-AUDIT. Completion requires recorded evidence; targets are not measured results. +## 8. Source-code retrieval + +- [x] Shared code classification, strict lexical/ANN filtering before top-k, + expanded source/build formats, bounded overlapping passages and actual lines. +- [x] Desktop Code scope, CLI `--code-only`, read-only MCP `code_only`, immutable + scope snapshots and code-format-preserving previews. +- [x] Google's CodeRetrieval query instruction, explicit worker capabilities, + task-scoped vector cache and genuine CPU prompt/debug parity. +- [x] Exact original source windows when Google's tokenizer normalizes text; + verified UTF-8 offsets, full tail coverage and actual token-budget checks. +- [x] Schema-11 migration schedules unchanged source for re-extraction; tests + preserve ordinary documents and filename availability through the upgrade. +- [x] Frozen 9-query real Google CPU regression with lexical comparison, + ordinary-document exclusion and exact MCP source excerpt verification. +- [ ] Evaluate independent real repositories, more languages, overloaded + symbols, near-duplicate routines and queries with no relevant implementation. +- [ ] Validate code search UI interaction on native Windows/macOS/Linux and + actual GPU hardware. Tiny CPU fixtures do not close those release gates. + ## Completed foundation details - SQLite FTS5 with exact filename priority, text snippets and strict filters. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 5c43771..298e815 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -7,10 +7,10 @@ and `SEARCHI_FORCE_CPU=1`. ## Current integration checkpoint (2026-10-12) -- Native desktop: **13/13 tests passed**, including structured trace/basic responses +- Native desktop: **15/15 tests passed**, including structured trace/basic responses and four optional-answer source/consent/revision tests. Post-generation cached query warmup is implemented but its interactive first-query gate remains open. - Frontend: **36/36 tests passed**, TypeScript/Vite production build and + Frontend: **39/39 tests passed**, TypeScript/Vite production build and formatting passed. Preview policy tests cover fractional timestamp seeking, image labels, one active/latest pending request, stale replies and Escape. - Semantic adapter: **8 default tests passed**; the explicit 48 MP resource test @@ -128,6 +128,19 @@ agent execution and release packaging remain open gates. ## Historical evidence +The source-code checkpoint uses the actual rebuilt/staged CPU Google worker: +9/9 frozen code queries rank their expected source file first (1/9 with lexical +retrieval alone), with exact MCP source excerpts and exclusion of a matching +ordinary document. This is a tiny synthetic regression, not a general code +benchmark. [Protocol and raw results](CODE-QUALITY.md). +Workspace checks passed 84 default tests; model/decoder/resource checks remain +opt-in. Separate actual CPU code tests verify task/debug vector behavior and +normalized-tokenizer source windows, including long Unicode code. The capture +backend's three tests, frontend build/format and strict Rust Clippy also pass. +Existing real Google text-reference and long-tail parity passed after the +tokenizer fix; regular document search retains its pinned reference behavior. +Native/GPU/platform and broad independent code-quality gates remain open. + The [EmbeddingGemma 2 audit](EMBEDDINGGEMMA2-AUDIT.md) records new CPU diagnostics: 24 uncached native queries per 1/2/4-thread setting, paired real MCP repeat/filter measurements, 20 unchanged result-set comparisons, actual Release startup/first diff --git a/docs/evidence/code-search-cpu.json b/docs/evidence/code-search-cpu.json new file mode 100644 index 0000000..77306fd --- /dev/null +++ b/docs/evidence/code-search-cpu.json @@ -0,0 +1,267 @@ +{ + "recorded_at": "2026-10-12T19:19:20.376773+00:00", + "device": "cpu", + "cpu_threads": 1, + "model": "google/embeddinggemma-2", + "precision": "verified Q8_0 weights, CPU inference", + "query_task": "task: code retrieval | query: ", + "scope": "8 tiny source files and 1 ordinary-document hard negative; synthetic regression, not general code accuracy", + "fixture_sha256": { + "a.py": "78d878583e1e29f8a33373e40e2541fef00ae706e1b01dd52b98e8a802e9def6", + "b.cpp": "9cdf9ba0abd4a7d068e37405bf40405c6f60cefa0cf83f072efb7419091acd39", + "c.ts": "6a23062539b6944767195506b086e7f0071e68ab636369a917efe4794abb61b4", + "d.py": "f2f221eab12c05e1a2da45c683e55d0d3bc8c934ebf393365b026ea181381b05", + "e.cs": "d05a8a3c8471263c9fc8ec940f482531a671b81cc518de6e43ed1e2c57fa3ee6", + "f.php": "41036ba20c6019cb94b7f7fde540f69449859ae19a6786751a6cfeec2880797e", + "g.py": "7bce1fb7efe687ad71ba63285a83d38fb322acdd0266d3ea59b25faaf96485c9", + "h.rs": "692577a259c1fafbc4285a63968294f05818944f2cd7aa17b0eb31b9b6254649", + "notes.txt": "2ab4d304fc6b0e03896a31cc409c27b0d458c172f6ffbc32960291c7fbd13f1f" + }, + "queries": [ + { + "mode": "lexical_only", + "query": "calculate a file's SHA256 without loading the entire file into memory", + "expected": "a.py", + "rank": null, + "results": [], + "roundtrip_ms": 8.568600023863837, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "find the first position where sorted numbers are greater than or equal to a target", + "expected": "b.cpp", + "rank": null, + "results": [], + "roundtrip_ms": 1.721800013910979, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "sum transaction amounts separately for each category", + "expected": "c.ts", + "rank": null, + "results": [], + "roundtrip_ms": 1.4463999832514673, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "retry failed requests with exponential backoff", + "expected": "d.py", + "rank": null, + "results": [], + "roundtrip_ms": 1.4552999928127974, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "release a semaphore even if the asynchronous operation throws", + "expected": "e.cs", + "rank": null, + "results": [], + "roundtrip_ms": 1.4591999934054911, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "escape unsafe HTML characters before displaying user input", + "expected": "f.php", + "rank": null, + "results": [], + "roundtrip_ms": 1.43420000676997, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "return the ten most expensive records", + "expected": "g.py", + "rank": null, + "results": [], + "roundtrip_ms": 1.2260999938007444, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "Datei prüfen ohne sie vollständig in den Speicher zu laden", + "expected": "a.py", + "rank": null, + "results": [], + "roundtrip_ms": 1.4165000175125897, + "source_verified": false + }, + { + "mode": "lexical_only", + "query": "refreshAccessToken", + "expected": "h.rs", + "rank": 1, + "results": [ + "h.rs" + ], + "roundtrip_ms": 1.4574000088032335, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "calculate a file's SHA256 without loading the entire file into memory", + "expected": "a.py", + "rank": 1, + "results": [ + "a.py", + "e.cs", + "d.py", + "h.rs", + "c.ts", + "b.cpp", + "f.php", + "g.py" + ], + "roundtrip_ms": 228.21230001864024, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "find the first position where sorted numbers are greater than or equal to a target", + "expected": "b.cpp", + "rank": 1, + "results": [ + "b.cpp", + "g.py", + "c.ts", + "e.cs", + "d.py", + "a.py", + "h.rs", + "f.php" + ], + "roundtrip_ms": 143.91000001342036, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "sum transaction amounts separately for each category", + "expected": "c.ts", + "rank": 1, + "results": [ + "c.ts", + "g.py", + "e.cs", + "h.rs", + "f.php", + "a.py", + "d.py", + "b.cpp" + ], + "roundtrip_ms": 109.81110000284389, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "retry failed requests with exponential backoff", + "expected": "d.py", + "rank": 1, + "results": [ + "d.py", + "h.rs", + "e.cs", + "g.py", + "a.py", + "c.ts", + "f.php", + "b.cpp" + ], + "roundtrip_ms": 126.85969998710789, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "release a semaphore even if the asynchronous operation throws", + "expected": "e.cs", + "rank": 1, + "results": [ + "e.cs", + "d.py", + "h.rs", + "f.php", + "a.py", + "c.ts", + "g.py", + "b.cpp" + ], + "roundtrip_ms": 121.69749999884516, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "escape unsafe HTML characters before displaying user input", + "expected": "f.php", + "rank": 1, + "results": [ + "f.php", + "d.py", + "e.cs", + "h.rs", + "c.ts", + "a.py", + "g.py", + "b.cpp" + ], + "roundtrip_ms": 100.53349999361672, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "return the ten most expensive records", + "expected": "g.py", + "rank": 1, + "results": [ + "g.py", + "c.ts", + "d.py", + "h.rs", + "e.cs", + "a.py", + "f.php", + "b.cpp" + ], + "roundtrip_ms": 85.61569999437779, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "Datei prüfen ohne sie vollständig in den Speicher zu laden", + "expected": "a.py", + "rank": 1, + "results": [ + "a.py", + "b.cpp", + "e.cs", + "d.py", + "h.rs", + "g.py", + "f.php", + "c.ts" + ], + "roundtrip_ms": 110.75749999145046, + "source_verified": true + }, + { + "mode": "hybrid", + "query": "refreshAccessToken", + "expected": "h.rs", + "rank": 1, + "results": [ + "h.rs", + "e.cs", + "d.py", + "c.ts", + "a.py", + "f.php", + "g.py", + "b.cpp" + ], + "roundtrip_ms": 63.88430000515655, + "source_verified": true + } + ] +} \ No newline at end of file diff --git a/skills/searchi-local-search/SKILL.md b/skills/searchi-local-search/SKILL.md index 66dd005..fc19b5d 100644 --- a/skills/searchi-local-search/SKILL.md +++ b/skills/searchi-local-search/SKILL.md @@ -38,3 +38,9 @@ and `--embedding-worker` flags only when actual hybrid assets are configured. Do not install or provision software/models without user authorization. CLI search has no source-handle tool; cite its actual returned snippet/location. See [configuration and tool contracts](../../docs/AGENT-INTEGRATION.md). + +For source-code retrieval, set MCP `code_only: true` or CLI `--code-only` so all +retrieval lanes filter to source files before ranking. Use exact symbols for +literal lookup and remembered behavior for a configured hybrid server. Cite +actual indexed line passages; do not claim AST, call-graph or definition-only +retrieval. See [code search](../../docs/CODE-SEARCH.md) for runnable examples. diff --git a/tools/code-eval/evaluate.py b/tools/code-eval/evaluate.py new file mode 100644 index 0000000..4bdf786 --- /dev/null +++ b/tools/code-eval/evaluate.py @@ -0,0 +1,96 @@ +"""Frozen tiny code-retrieval regression; genuine Google worker, CPU only. + +Generated public fixtures, not a production accuracy benchmark. Run from repo +after rebuilding CLI and CPU worker. No private folders are indexed. +""" +import hashlib +from datetime import datetime, timezone +import importlib.util +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("benchmark", ROOT / "tools/embeddinggemma2/benchmark.py") +BENCH = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BENCH) + +FILES = { + "a.py": "import hashlib\ndef process(path):\n h = hashlib.sha256()\n with open(path, 'rb') as stream:\n for chunk in iter(lambda: stream.read(65536), b''):\n h.update(chunk)\n return h.hexdigest()\n", + "b.cpp": "#include \nint locate(const std::vector& v, int x) {\n int lo = 0, hi = v.size();\n while (lo < hi) {\n int mid = lo + (hi-lo)/2;\n if (v[mid] < x) lo = mid+1; else hi = mid;\n }\n return lo;\n}\n", + "c.ts": "export function process(rows: {category: string; amount: number}[]) {\n const totals = new Map();\n for (const row of rows) totals.set(row.category, (totals.get(row.category) ?? 0) + row.amount);\n return totals;\n}\n", + "d.py": "import time\nimport urllib.request\ndef process(url):\n for attempt in range(5):\n try:\n return urllib.request.urlopen(url, timeout=10).read()\n except OSError:\n if attempt == 4: raise\n time.sleep(0.2 * 2 ** attempt)\n", + "e.cs": "using System;\nusing System.Threading;\nusing System.Threading.Tasks;\npublic static class Module {\n public static async Task Process(SemaphoreSlim gate, Func work) {\n await gate.WaitAsync();\n try { await work(); } finally { gate.Release(); }\n }\n}\n", + "f.php": " String {\n format!(\"renewed:{}\", previous)\n}\n", + "notes.txt": "refreshAccessToken retry failed requests with exponential backoff compute a file digest sum amounts by category escape HTML sorted numbers", +} +QUERIES = [ + ("calculate a file's SHA256 without loading the entire file into memory", "a.py"), + ("find the first position where sorted numbers are greater than or equal to a target", "b.cpp"), + ("sum transaction amounts separately for each category", "c.ts"), + ("retry failed requests with exponential backoff", "d.py"), + ("release a semaphore even if the asynchronous operation throws", "e.cs"), + ("escape unsafe HTML characters before displaying user input", "f.php"), + ("return the ten most expensive records", "g.py"), + ("Datei prüfen ohne sie vollständig in den Speicher zu laden", "a.py"), + ("refreshAccessToken", "h.rs"), +] + + +def main(): + output = ROOT / ".searchi/code-eval" + corpus = output / "corpus" + corpus.mkdir(parents=True, exist_ok=True) + for name, content in FILES.items(): + (corpus / name).write_text(content, encoding="utf-8", newline="\n") + cli = ROOT / "target/release" / ("searchi.exe" if os.name == "nt" else "searchi") + models = ROOT / ".searchi/models/embeddinggemma2" + worker = ROOT / ".searchi/runtime/cpu/Release" / ("searchi-gemma-worker.exe" if os.name == "nt" else "searchi-gemma-worker") + db = output / "index.db" + environment = dict(os.environ, SEARCHI_FORCE_CPU="1") + def run(*args): + subprocess.run([str(cli), *map(str, args)], env=environment, check=True) + run("index", "--db", db, "--root", corpus) + run("embed", "--db", db, "--model-dir", models, "--embedding-worker", worker, "--text-only") + rows = [] + for mode in ("lexical_only", "hybrid"): + command = [str(cli), "mcp", "--db", str(db)] + if mode == "hybrid": + command += ["--model-dir", str(models), "--embedding-worker", str(worker)] + session = BENCH.Session(command) + try: + session.mcp(1, "initialize", {"protocolVersion": "2025-03-26", "capabilities": {}, "clientInfo": {"name": "code-eval", "version": "1"}}) + session.send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + for number, (query, expected) in enumerate(QUERIES, 2): + result, duration = session.mcp(number, "tools/call", {"name": "search", "arguments": {"query": query, "code_only": True, "limit": 10}}) + assert not result.get("isError"), result + data = result["structuredContent"] + assert data["mode"] == mode and data["code_only"] + names = [r["filename"] for r in data["results"]] + assert "notes.txt" not in names + rank = names.index(expected) + 1 if expected in names else None + source_verified = False + if rank is not None: + matched = data["results"][rank - 1] + source, _ = session.mcp(number + 100, "tools/call", {"name": "read_source", "arguments": {"handle": matched["source_handle"]}}) + assert not source.get("isError"), source + source = source["structuredContent"] + assert source["source"] == {"kind": "line", "number": 1} + assert source["text"] == FILES[expected] + source_verified = True + rows.append({"mode": mode, "query": query, "expected": expected, "rank": rank, "results": names, "roundtrip_ms": duration, "source_verified": source_verified}) + finally: + session.close() + report = {"recorded_at": datetime.now(timezone.utc).isoformat(), "device": "cpu", "cpu_threads": 1, "model": "google/embeddinggemma-2", "precision": "verified Q8_0 weights, CPU inference", "query_task": "task: code retrieval | query: ", "scope": "8 tiny source files and 1 ordinary-document hard negative; synthetic regression, not general code accuracy", "fixture_sha256": {n: hashlib.sha256(c.encode()).hexdigest() for n, c in FILES.items()}, "queries": rows} + (output / "results.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + for mode in ("lexical_only", "hybrid"): + selected = [r for r in rows if r["mode"] == mode] + print(mode, "top1", sum(r["rank"] == 1 for r in selected), "/", len(selected), "top3", sum(r["rank"] is not None and r["rank"] <= 3 for r in selected)) + print("Recorded", output / "results.json") + + +if __name__ == "__main__": + main() diff --git a/tools/desktop-capture/backend/src/main.rs b/tools/desktop-capture/backend/src/main.rs index 5b20ef8..4c04ae3 100644 --- a/tools/desktop-capture/backend/src/main.rs +++ b/tools/desktop-capture/backend/src/main.rs @@ -35,6 +35,7 @@ struct Args { #[derive(Default, Deserialize)] #[serde(deny_unknown_fields)] struct Filters { + code_only: Option, folder: Option, extension: Option, min_size: Option, @@ -53,6 +54,7 @@ impl Filters { ); } Ok(SearchOptions { + code_only: self.code_only.unwrap_or(false), folder: self.folder, extension: self.extension, min_size: self.min_size,