feat: add Google code retrieval across desktop CLI and MCP
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
Core CI / core (ubuntu-latest) (push) Canceled after 2m23s

This commit is contained in:
karl committed 2026-10-12 21:22:11 +02:00
1 parent 8b9b079469
commit 81846b3e98
32 files changed
+1303 -59

No files matched your search

+81 -2
View File
@@ -28,6 +28,7 @@ struct FilterKey {
min_duration: Option<u32>,
max_duration: Option<u32>,
source_type: Option<String>,
code_only: bool,
}
struct EligibilityCache {
@@ -551,6 +552,7 @@ impl Store {
min_duration: options.min_duration_ms,
max_duration: options.max_duration_ms,
source_type: source_type.map(str::to_owned),
code_only: options.code_only,
};
trace.end(filter_span, Default::default());
let eligibility_span = trace.begin("semantic.eligibility", parent);
@@ -906,7 +908,12 @@ fn collect_eligibility(
filter: &FilterKey,
path: Option<&str>,
) -> Result<EligibilityCache> {
let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
let code = if filter.code_only {
format!(" AND {}", searchi_core::code_file_predicate())
} else {
String::new()
};
let mut statement=connection.prepare(&format!("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime))
AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
@@ -915,7 +922,7 @@ fn collect_eligibility(
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
AND (?12 IS NULL OR c.source_type=?12)
AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?;
let mut rows = statement.query(params![
generation,
filter.folder,
@@ -971,6 +978,78 @@ mod tests {
use searchi_core::SearchIndex;
const DIMENSIONS: usize = 768;
#[test]
fn code_filter_precedes_ann_top_k_and_cache_distinguishes_scope() -> Result<()> {
// Retrieval geometry only: production vectors come from Google.
let directory = tempfile::tempdir()?;
let root = directory.path().join("sources");
std::fs::create_dir(&root)?;
for n in 0..40 {
std::fs::write(
root.join(format!("document-{n}.md")),
"parse JSON and retry requests",
)?;
}
std::fs::write(
root.join("network.rs"),
"fn retry_request() { parse_json(); }",
)?;
std::fs::write(root.join("Makefile"), "build: compile_sources\n")?;
let db = directory.path().join("index.db");
let mut lexical = SearchIndex::open(&db)?;
lexical.index_folder(&root)?;
let mut store = Store::open(&db, DIMENSIONS)?;
let generation = store.begin("code-filter-proof")?;
let mut query = vec![0.; DIMENSIONS];
query[0] = 1.;
let mut weaker = query.clone();
weaker[0] = 0.9;
weaker[1] = (1.0_f32 - 0.9_f32.powi(2)).sqrt();
for passage in lexical.passage_batch(0, 100)? {
let vector = if searchi_core::is_code_path(Path::new(&passage.path)) {
weaker.clone()
} else {
query.clone()
};
let window = TextWindow {
text: passage.text.clone(),
start_byte: 0,
end_byte: passage.text.len(),
};
store.insert(generation, &passage, 0, &[window], &[vector])?;
}
store.activate(generation)?;
assert!(
store.search(&query, "code-filter-proof", 1, &SearchOptions::default())?[0]
.filename
.ends_with(".md")
);
let code = SearchOptions {
code_only: true,
..Default::default()
};
let hits = store.search(&query, "code-filter-proof", 2, &code)?;
assert_eq!(hits.len(), 2);
assert!(hits
.iter()
.all(|h| searchi_core::is_code_path(Path::new(&h.path))));
let incompatible = SearchOptions {
code_only: true,
extension: Some("md".into()),
..Default::default()
};
assert!(store
.search(&query, "code-filter-proof", 2, &incompatible)?
.is_empty());
std::fs::remove_file(root.join("network.rs"))?;
lexical.index_folder(&root)?;
assert_eq!(
store.search(&query, "code-filter-proof", 2, &code)?.len(),
1
);
Ok(())
}
#[test]
fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> {
// Geometry verifies grouped retrieval, not model inference or codecs.