feat: add Google code retrieval across desktop CLI and MCP
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
Core CI / core (ubuntu-latest) (push) Canceled after 2m23s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
Core CI / core (ubuntu-latest) (push) Canceled after 2m23s
This commit is contained in:
1 parent
8b9b079469
commit
81846b3e98
32 files changed
+1303
-59
No files matched your search
@@ -28,6 +28,7 @@ struct FilterKey {
|
||||
min_duration: Option<u32>,
|
||||
max_duration: Option<u32>,
|
||||
source_type: Option<String>,
|
||||
code_only: bool,
|
||||
}
|
||||
|
||||
struct EligibilityCache {
|
||||
@@ -551,6 +552,7 @@ impl Store {
|
||||
min_duration: options.min_duration_ms,
|
||||
max_duration: options.max_duration_ms,
|
||||
source_type: source_type.map(str::to_owned),
|
||||
code_only: options.code_only,
|
||||
};
|
||||
trace.end(filter_span, Default::default());
|
||||
let eligibility_span = trace.begin("semantic.eligibility", parent);
|
||||
@@ -906,7 +908,12 @@ fn collect_eligibility(
|
||||
filter: &FilterKey,
|
||||
path: Option<&str>,
|
||||
) -> Result<EligibilityCache> {
|
||||
let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
|
||||
let code = if filter.code_only {
|
||||
format!(" AND {}", searchi_core::code_file_predicate())
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let mut statement=connection.prepare(&format!("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
|
||||
LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
|
||||
JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime))
|
||||
AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
|
||||
@@ -915,7 +922,7 @@ fn collect_eligibility(
|
||||
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
|
||||
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
|
||||
AND (?12 IS NULL OR c.source_type=?12)
|
||||
AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
|
||||
AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?;
|
||||
let mut rows = statement.query(params![
|
||||
generation,
|
||||
filter.folder,
|
||||
@@ -971,6 +978,78 @@ mod tests {
|
||||
use searchi_core::SearchIndex;
|
||||
const DIMENSIONS: usize = 768;
|
||||
|
||||
#[test]
|
||||
fn code_filter_precedes_ann_top_k_and_cache_distinguishes_scope() -> Result<()> {
|
||||
// Retrieval geometry only: production vectors come from Google.
|
||||
let directory = tempfile::tempdir()?;
|
||||
let root = directory.path().join("sources");
|
||||
std::fs::create_dir(&root)?;
|
||||
for n in 0..40 {
|
||||
std::fs::write(
|
||||
root.join(format!("document-{n}.md")),
|
||||
"parse JSON and retry requests",
|
||||
)?;
|
||||
}
|
||||
std::fs::write(
|
||||
root.join("network.rs"),
|
||||
"fn retry_request() { parse_json(); }",
|
||||
)?;
|
||||
std::fs::write(root.join("Makefile"), "build: compile_sources\n")?;
|
||||
let db = directory.path().join("index.db");
|
||||
let mut lexical = SearchIndex::open(&db)?;
|
||||
lexical.index_folder(&root)?;
|
||||
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||
let generation = store.begin("code-filter-proof")?;
|
||||
let mut query = vec![0.; DIMENSIONS];
|
||||
query[0] = 1.;
|
||||
let mut weaker = query.clone();
|
||||
weaker[0] = 0.9;
|
||||
weaker[1] = (1.0_f32 - 0.9_f32.powi(2)).sqrt();
|
||||
for passage in lexical.passage_batch(0, 100)? {
|
||||
let vector = if searchi_core::is_code_path(Path::new(&passage.path)) {
|
||||
weaker.clone()
|
||||
} else {
|
||||
query.clone()
|
||||
};
|
||||
let window = TextWindow {
|
||||
text: passage.text.clone(),
|
||||
start_byte: 0,
|
||||
end_byte: passage.text.len(),
|
||||
};
|
||||
store.insert(generation, &passage, 0, &[window], &[vector])?;
|
||||
}
|
||||
store.activate(generation)?;
|
||||
assert!(
|
||||
store.search(&query, "code-filter-proof", 1, &SearchOptions::default())?[0]
|
||||
.filename
|
||||
.ends_with(".md")
|
||||
);
|
||||
let code = SearchOptions {
|
||||
code_only: true,
|
||||
..Default::default()
|
||||
};
|
||||
let hits = store.search(&query, "code-filter-proof", 2, &code)?;
|
||||
assert_eq!(hits.len(), 2);
|
||||
assert!(hits
|
||||
.iter()
|
||||
.all(|h| searchi_core::is_code_path(Path::new(&h.path))));
|
||||
let incompatible = SearchOptions {
|
||||
code_only: true,
|
||||
extension: Some("md".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(store
|
||||
.search(&query, "code-filter-proof", 2, &incompatible)?
|
||||
.is_empty());
|
||||
std::fs::remove_file(root.join("network.rs"))?;
|
||||
lexical.index_folder(&root)?;
|
||||
assert_eq!(
|
||||
store.search(&query, "code-filter-proof", 2, &code)?.len(),
|
||||
1
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> {
|
||||
// Geometry verifies grouped retrieval, not model inference or codecs.
|
||||
|
||||
Reference in new issue
Block a user