karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

perf: index filename candidates and avoid full-corpus retrieval scans
Core CI / core (ubuntu-latest) (push) Failing after 1m18s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 3m34s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s

Browse Source
This commit is contained in:
karl committed 2026-10-12 16:35:46 +02:00
1 parent a9b6592330
commit 55fa751571
11 files changed
+1035 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/searchi-core/src/lib.rs
+290 -64
View File
@@ -5,16 +5,18 @@
//! Worker isolation remains a production milestone.
mod extraction;
use anyhow::{bail, Context, Result};
use rusqlite::{params, Connection, OptionalExtension};
use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExtension};
use serde::Serialize;
use std::{
cell::RefCell,
collections::{BTreeSet, HashMap},
fs,
path::{Path, PathBuf},
time::UNIX_EPOCH,
};
use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 4;
const SCHEMA_VERSION: i64 = 7;
const MAX_RESULTS: usize = 1000;
#[derive(Debug, Default, Serialize)]
@@ -92,6 +94,7 @@ pub struct IndexStats {
pub struct SearchIndex {
connection: Connection,
database_path: Option<PathBuf>,
gram_frequencies: RefCell<HashMap<String, i64>>,
}
impl SearchIndex {
@@ -159,16 +162,66 @@ impl SearchIndex {
CREATE TRIGGER IF NOT EXISTS files_passages_delete AFTER DELETE ON files BEGIN
DELETE FROM passages WHERE file_path=old.path; END;",
)?;
if version != 4 {
if version > 0 && version < 4 {
// Re-extract records from older schemas to populate genuine locations.
initialization.execute("UPDATE files SET mtime=-1", [])?;
}
initialization.execute_batch("PRAGMA user_version=4;")?;
if version < 5 {
initialization.execute_batch("CREATE TABLE passages_new(
id INTEGER PRIMARY KEY, text TEXT NOT NULL, file_path TEXT NOT NULL,
kind TEXT NOT NULL, number INTEGER NOT NULL);
INSERT INTO passages_new SELECT rowid,text,file_path,kind,number FROM passages;
DROP TRIGGER files_passages_delete;
DROP TABLE passages;
ALTER TABLE passages_new RENAME TO passages;
CREATE TRIGGER files_passages_delete AFTER DELETE ON files BEGIN
DELETE FROM passages WHERE file_path=old.path; END;
CREATE INDEX passages_file ON passages(file_path,id);
CREATE VIRTUAL TABLE passage_search USING fts5(text,content='passages',content_rowid='id',tokenize='unicode61');
CREATE TRIGGER passages_insert AFTER INSERT ON passages BEGIN
INSERT INTO passage_search(rowid,text) VALUES(new.id,new.text); END;
CREATE TRIGGER passages_delete AFTER DELETE ON passages BEGIN
INSERT INTO passage_search(passage_search,rowid,text) VALUES('delete',old.id,old.text); END;
CREATE TRIGGER passages_update AFTER UPDATE ON passages BEGIN
INSERT INTO passage_search(passage_search,rowid,text) VALUES('delete',old.id,old.text);
INSERT INTO passage_search(rowid,text) VALUES(new.id,new.text); END;
INSERT INTO passage_search(passage_search) VALUES('rebuild');
CREATE INDEX files_filename ON files(lower(filename),id);
CREATE VIRTUAL TABLE names USING fts5(filename,path,content='files',content_rowid='id',tokenize='trigram');
CREATE TRIGGER files_names_insert AFTER INSERT ON files BEGIN
INSERT INTO names(rowid,filename,path) VALUES(new.id,new.filename,new.path); END;
CREATE TRIGGER files_names_delete AFTER DELETE ON files BEGIN
INSERT INTO names(names,rowid,filename,path) VALUES('delete',old.id,old.filename,old.path); END;
CREATE TRIGGER files_names_update AFTER UPDATE ON files BEGIN
INSERT INTO names(names,rowid,filename,path) VALUES('delete',old.id,old.filename,old.path);
INSERT INTO names(rowid,filename,path) VALUES(new.id,new.filename,new.path); END;
INSERT INTO names(names) VALUES('rebuild');
PRAGMA user_version=5;")?;
}
if version < 6 {
initialization.execute_batch("CREATE VIRTUAL TABLE content_search USING fts5(content,
content='files',content_rowid='id',tokenize='unicode61');
CREATE TRIGGER files_content_insert AFTER INSERT ON files WHEN new.content!='' BEGIN
INSERT INTO content_search(rowid,content) VALUES(new.id,new.content); END;
CREATE TRIGGER files_content_delete AFTER DELETE ON files WHEN old.content!='' BEGIN
INSERT INTO content_search(content_search,rowid,content) VALUES('delete',old.id,old.content); END;
CREATE TRIGGER files_content_update AFTER UPDATE ON files BEGIN
INSERT INTO content_search(content_search,rowid,content)
SELECT 'delete',old.id,old.content WHERE old.content!='';
INSERT INTO content_search(rowid,content) SELECT new.id,new.content WHERE new.content!=''; END;
INSERT INTO content_search(rowid,content) SELECT id,content FROM files WHERE content!='';
PRAGMA user_version=6;")?;
}
initialization.execute_batch(
"CREATE VIRTUAL TABLE IF NOT EXISTS names_vocab USING fts5vocab(names,'row');
PRAGMA user_version=7;",
)?;
initialization.commit()?;
}
Ok(Self {
connection,
database_path,
gram_frequencies: RefCell::new(HashMap::new()),
})
}
@@ -300,13 +353,14 @@ impl SearchIndex {
)?;
transaction.execute("DELETE FROM seen_paths", [])?;
transaction.commit()?;
self.gram_frequencies.borrow_mut().clear();
Ok(report)
}
/// Literal input is escaped before FTS parsing; user text cannot inject operators.
/// Exact filenames precede partial filenames, which precede content matches.
/// Filename substring matching currently scans metadata; a dedicated substring
/// index is required before claiming million-file latency targets.
/// Substrings of at least three Unicode characters use a trigram index.
/// Shorter substrings retain complete matching using a slower metadata scan.
pub fn search(&self, query: &str, limit: usize) -> Result<Vec<SearchResult>> {
self.search_with_options(query, limit, &SearchOptions::default())
}
@@ -370,65 +424,237 @@ impl SearchIndex {
.map(|term| format!("\"{}\"", term.replace('"', "\"\"")))
.collect::<Vec<_>>()
.join(" AND ");
let mut statement = self.connection.prepare(
"WITH hits AS (
SELECT rowid AS id, bm25(search, 8.0, 2.0, 1.0) AS rank,
snippet(search, 2, '', '', ' … ', 24) AS excerpt
FROM search WHERE search MATCH ?1)
SELECT f.path, f.filename, COALESCE(h.excerpt, ''), COALESCE(h.rank, 0),
CASE WHEN lower(f.filename)=lower(?2) THEN 0
WHEN instr(lower(f.filename), lower(?2)) > 0 THEN 1 ELSE 2 END AS tier, f.extraction_error
FROM files f LEFT JOIN hits h ON h.id=f.id
WHERE (h.id IS NOT NULL OR instr(lower(f.filename), lower(?2)) > 0)
AND (?4 IS NULL OR substr(f.path,1,length(?4))=?4)
AND (?5 IS NULL OR f.extension=?5)
AND (?6 IS NULL OR f.size>=?6) AND (?7 IS NULL OR f.size<=?7)
AND (?8 IS NULL OR f.mtime/1000000>=?8) AND (?9 IS NULL OR f.mtime/1000000<=?9)
ORDER BY tier, h.rank, f.path LIMIT ?3",
)?;
let mut results = statement
.query_map(
params![
fts,
query,
limit.min(MAX_RESULTS) as i64,
folder_prefix,
extension,
min_size,
max_size,
options.modified_after_unix_ms,
options.modified_before_unix_ms
],
|row| {
let tier: i64 = row.get(4)?;
Ok(SearchResult {
path: row.get(0)?,
filename: row.get(1)?,
snippet: row.get(2)?,
score: -row.get::<_, f64>(3)?,
match_kind: match tier {
0 => "exact_filename",
1 => "filename",
_ => "content",
}
.to_owned(),
source: None,
extraction_error: row.get(5)?,
})
let limit = limit.min(MAX_RESULTS);
let filters = " AND (?3 IS NULL OR substr(f.path,1,length(?3))=?3)
AND (?4 IS NULL OR f.extension=?4)
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)";
let filter_values = [
folder_prefix.map_or(Value::Null, Value::Text),
extension.map_or(Value::Null, Value::Text),
min_size.map_or(Value::Null, Value::Integer),
max_size.map_or(Value::Null, Value::Integer),
options
.modified_after_unix_ms
.map_or(Value::Null, Value::Integer),
options
.modified_before_unix_ms
.map_or(Value::Null, Value::Integer),
];
let mut results = Vec::new();
let mut selected = Vec::<i64>::new();
// Each lane filters its entire candidate set before LIMIT. There is no
// hidden pre-filter candidate cap that can hide later matching files.
let trigram = query.chars().count() >= 3;
// A long FTS phrase may walk common leading trigram postings even when
// later trigrams are rare. Use the rarest known trigram as a candidate
// superset, then verify the whole literal substring. The 256-term lookup
// budget can only enlarge candidates; it never caps results. Frequency
// lookups are cached because FTS vocabulary counts have a first-use cost.
let characters = query.chars().collect::<Vec<_>>();
let grams = characters
.windows(3)
.map(|window| window.iter().collect::<String>())
.collect::<BTreeSet<_>>()
.into_iter()
.take(256)
.collect::<Vec<_>>();
let mut vocabulary = self
.connection
.prepare("SELECT doc FROM names_vocab WHERE term=?1")?;
let mut cache = self.gram_frequencies.borrow_mut();
if cache.len() + grams.len() > 4096 {
cache.clear();
}
let mut rare_first = grams
.into_iter()
.map(|gram| {
let term = gram.to_lowercase();
let frequency = if let Some(frequency) = cache.get(&term) {
*frequency
} else {
let frequency = vocabulary
.query_row([&term], |row| row.get::<_, i64>(0))
.optional()?
.unwrap_or(0);
cache.insert(term, frequency);
frequency
};
Ok((frequency, gram))
})
.collect::<Result<Vec<_>>>()?;
rare_first.sort();
drop(cache);
let grams = rare_first
.into_iter()
.take(1)
.map(|(_, gram)| format!("\"{}\"", gram.replace('"', "\"\"")))
.collect::<Vec<_>>()
.join(" AND ");
let mut lanes = vec![
(
"exact_filename",
"files f".to_owned(),
"lower(f.filename)=lower(?1)".to_owned(),
query.to_owned(),
"f.id",
),
(
"filename",
if trigram {
"names CROSS JOIN files f ON f.id=names.rowid".into()
} else {
"files f".into()
},
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut passages = self.connection.prepare("SELECT kind,number,snippet(passages,0,'','',' … ',24)
FROM passages WHERE passages MATCH ?1 AND file_path=?2 ORDER BY bm25(passages),rowid LIMIT 1")?;
for result in &mut results {
let passage: Option<(String, u32, String)> = passages
.query_row(params![fts, result.path], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
})
.optional()?;
if let Some((kind, number, snippet)) = passage {
result.source = Some(SourceLocation { kind, number });
result.snippet = snippet;
if trigram {
"names MATCH ?1 AND instr(lower(f.filename),lower(?2))>0 AND lower(f.filename)!=lower(?2)".into()
} else {
"instr(lower(f.filename),lower(?1))>0 AND lower(f.filename)!=lower(?2)".into()
},
if trigram {
format!("filename:({grams})")
} else {
query.into()
},
if trigram { "names.rowid" } else { "f.id" },
),
(
"path",
if trigram {
"names CROSS JOIN files f ON f.id=names.rowid".into()
} else {
"files f".into()
},
if trigram {
"names MATCH ?1 AND instr(lower(f.path),lower(?2))>0 AND instr(lower(f.filename),lower(?2))=0".into()
} else {
"instr(lower(f.path),lower(?1))>0 AND instr(lower(f.filename),lower(?2))=0"
.into()
},
if trigram {
format!("path:({grams})")
} else {
query.into()
},
if trigram { "names.rowid" } else { "f.id" },
),
(
"content",
"content_search CROSS JOIN files f ON f.id=content_search.rowid".into(),
"content_search MATCH ?1".into(),
fts.clone(),
"rank,f.id",
),
];
// Single-token names/identifiers use literal substring matching. A
// multiword query can also match token-separated names such as annual_report.
if query.split_whitespace().count() > 1 {
lanes.push((
"name_terms",
"search CROSS JOIN files f ON f.id=search.rowid".into(),
"search MATCH ?1 AND rank MATCH 'bm25(8.0,2.0,1.0)'".into(),
fts.clone(),
"rank,f.id",
));
}
for (kind, from, condition, expression, ordering) in lanes {
if results.len() == limit {
break;
}
let exclusions = if selected.is_empty() {
String::new()
} else {
format!(
" AND f.id NOT IN ({})",
selected
.iter()
.map(i64::to_string)
.collect::<Vec<_>>()
.join(",")
)
};
let rank = if matches!(kind, "content" | "name_terms") {
"rank"
} else {
"0.0"
};
let snippet = if kind == "content" {
"snippet(content_search,0,'','',' … ',24)"
} else if kind == "name_terms" {
"snippet(search,2,'','',' … ',24)"
} else {
"''"
};
let sql = format!(
"SELECT f.id,f.path,f.filename,{snippet},{rank},f.extraction_error FROM {from}
WHERE {condition} {filters} {exclusions} ORDER BY {ordering} LIMIT ?9"
);
let mut values = vec![Value::Text(expression), Value::Text(query.into())];
values.extend(filter_values.iter().cloned());
values.push(Value::Integer((limit - results.len()) as i64));
let mut statement = self.connection.prepare(&sql)?;
let matches = statement
.query_map(params_from_iter(values), |row| {
Ok((
row.get::<_, i64>(0)?,
SearchResult {
path: row.get(1)?,
filename: row.get(2)?,
snippet: row.get(3)?,
score: -row.get::<_, f64>(4)?,
match_kind: kind.into(),
source: None,
extraction_error: row.get(5)?,
},
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
for (id, result) in matches {
selected.push(id);
results.push(result);
}
}
if !results.is_empty() {
// Ownership has a B-tree index. The rowid IN constraint intersects
// the FTS postings with only the actual finalists' passage IDs.
let placeholders = (0..results.len())
.map(|_| "?")
.collect::<Vec<_>>()
.join(",");
let sql = format!("WITH matches AS MATERIALIZED (
SELECT p.file_path,p.kind,p.number,p.id,snippet(passage_search,0,'','',' … ',24) AS excerpt,
bm25(passage_search) AS relevance
FROM passage_search JOIN passages p ON p.id=passage_search.rowid
WHERE passage_search MATCH ? AND passage_search.rowid IN (
SELECT id FROM passages WHERE file_path IN ({placeholders})))
SELECT file_path,kind,number,excerpt FROM (
SELECT *,row_number() OVER(PARTITION BY file_path ORDER BY relevance,id) AS ordinal FROM matches)
WHERE ordinal=1");
let values = std::iter::once(Value::Text(fts)).chain(
results
.iter()
.map(|result| Value::Text(result.path.clone())),
);
let mut statement = self.connection.prepare(&sql)?;
let sources = statement
.query_map(params_from_iter(values), |row| {
Ok((
row.get::<_, String>(0)?,
(
SourceLocation {
kind: row.get(1)?,
number: row.get(2)?,
},
row.get::<_, String>(3)?,
),
))
})?
.collect::<rusqlite::Result<HashMap<_, _>>>()?;
for result in &mut results {
if let Some((source, snippet)) = sources.get(&result.path) {
result.source = Some(source.clone());
result.snippet = snippet.clone();
}
}
}
Ok(results)