karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Prepare text search during media setup and retain query sessions through promotion

Browse Source
This commit is contained in:
karl committed 2026-10-13 09:27:40 +02:00
1 parent af6ce29125
commit 9aae3f8216
15 files changed
+1235 -65
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/searchi-service/src/lib.rs
+100 -11
View File
@@ -53,6 +53,42 @@ fn reusable_policy(expected: &str, active: &str) -> bool {
matches!(values,Ok(v) if v.len()==8 && v[0]>=2 && (5000..=30000).contains(&v[3]) && v[4]<v[3] && v[5]>0 && v[6]<=u32::MAX as u64)
}
/// Resolve compatible publication coverage without broadening model or window
/// policy compatibility. A retained reader can start under either coverage.
fn policy_base(policy: &str) -> &str {
if let Some(base) = policy.strip_suffix(":text-only") {
return base;
}
if let Some((base, _)) = policy.rsplit_once(":custom-") {
if reusable_policy(base, policy) {
return base;
}
}
policy
}
fn compatible_query_policy(expected: &str, active: &str) -> bool {
reusable_policy(policy_base(expected), active)
}
fn policy_coverage(expected: &str, active: &str) -> Option<SemanticCoverage> {
compatible_query_policy(expected, active).then(|| {
if active.ends_with(":text-only") {
SemanticCoverage::TextOnly
} else {
SemanticCoverage::FullMedia
}
})
}
/// Requested indexing coverage is separate from warmed model readiness.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SemanticCoverage {
TextOnly,
FullMedia,
}
/// Model-specific query/document prompts and actual tokenizer windows belong
/// to the provider. Storage and ranking never assume a particular dimension.
pub trait EmbeddingProvider: Send {
@@ -181,6 +217,11 @@ pub struct EmbeddingReport {
#[derive(Debug, Serialize)]
pub struct EmbeddingStatus {
/// Coverage of a compatible, prepared active generation, if any.
pub coverage: Option<SemanticCoverage>,
pub requested_coverage: SemanticCoverage,
/// Early setup must preserve an existing compatible full generation.
pub text_indexing_allowed: bool,
pub failures: Vec<MediaFailure>,
pub generation: Option<i64>,
pub chunks: usize,
@@ -236,18 +277,31 @@ impl HybridSearch {
/// Cheap local status check. No model, runtime worker, or network is loaded.
/// Corpus revision triggers detect file inserts, updates, and deletions.
pub fn embedding_status(db: impl AsRef<Path>) -> Result<EmbeddingStatus> {
Self::embedding_status_for_coverage(db, SemanticCoverage::FullMedia)
}
pub fn embedding_status_for_coverage(
db: impl AsRef<Path>,
requested_coverage: SemanticCoverage,
) -> Result<EmbeddingStatus> {
let connection =
rusqlite::Connection::open_with_flags(db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
connection.busy_timeout(std::time::Duration::from_secs(5))?;
let exists: bool = connection.query_row(
let snapshot = connection.unchecked_transaction()?;
let exists: bool = snapshot.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='semantic_meta')",
[],
|row| row.get(0),
)?;
let pending: usize =
connection.query_row("SELECT (SELECT count(*) FROM passages)+(SELECT count(*) FROM files WHERE extension IN ('png','jpg','jpeg','webp','bmp','mp4','mkv','mov','avi','webm','m4v','mpeg','mpg','mp3','flac','wav','m4a','ogg','opus','aac','aiff','aif'))", [], |row| row.get(0))?;
let pending: usize = match requested_coverage {
SemanticCoverage::TextOnly => snapshot.query_row("SELECT count(*) FROM passages", [], |row| row.get(0))?,
SemanticCoverage::FullMedia => snapshot.query_row("SELECT (SELECT count(*) FROM passages)+(SELECT count(*) FROM files WHERE extension IN ('png','jpg','jpeg','webp','bmp','mp4','mkv','mov','avi','webm','m4v','mpeg','mpg','mp3','flac','wav','m4a','ogg','opus','aac','aiff','aif'))", [], |row| row.get(0))?,
};
if !exists {
return Ok(EmbeddingStatus {
coverage: None,
requested_coverage,
text_indexing_allowed: true,
failures: Vec::new(),
generation: None,
chunks: 0,
@@ -256,13 +310,16 @@ impl HybridSearch {
pending,
});
}
let version: i64 = connection.query_row(
let version: i64 = snapshot.query_row(
"SELECT schema_version FROM semantic_meta WHERE singleton=1",
[],
|row| row.get(0),
)?;
if version < 6 {
return Ok(EmbeddingStatus {
coverage: None,
requested_coverage,
text_indexing_allowed: true,
failures: Vec::new(),
generation: None,
chunks: 0,
@@ -275,15 +332,15 @@ impl HybridSearch {
(6..=8).contains(&version),
"Unsupported semantic schema version {version}"
);
let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
"SELECT g.id,g.fingerprint,CASE WHEN g.ann_blob IS NOT NULL AND g.dimensions=768 THEN g.corpus_revision END,m.corpus_revision FROM semantic_meta m LEFT JOIN semantic_generations g ON g.id=m.active_generation WHERE m.singleton=1",[],
|row| Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?)))?;
let chunks: usize = connection.query_row(
let (generation,fingerprint,revision,current,dimensions):(Option<i64>,Option<String>,Option<i64>,i64,Option<usize>)=snapshot.query_row(
"SELECT g.id,g.fingerprint,CASE WHEN g.ann_blob IS NOT NULL AND g.dimensions=768 THEN g.corpus_revision END,m.corpus_revision,g.dimensions FROM semantic_meta m LEFT JOIN semantic_generations g ON g.id=m.active_generation WHERE m.singleton=1",[],
|row| Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?,row.get(4)?)))?;
let chunks: usize = snapshot.query_row(
"SELECT count(*) FROM semantic_chunks WHERE generation=?1",
[generation],
|row| row.get(0),
)?;
let mut failures_query=connection.prepare("SELECT path,message FROM semantic_media_errors WHERE generation=?1 ORDER BY path LIMIT 1000")?;
let mut failures_query=snapshot.prepare("SELECT path,message FROM semantic_media_errors WHERE generation=?1 ORDER BY path LIMIT 1000")?;
let failures = failures_query
.query_map([generation], |r| {
Ok(MediaFailure {
@@ -292,9 +349,25 @@ impl HybridSearch {
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
let stale = revision != Some(current)
|| fingerprint.as_deref() != Some(generation_fingerprint(GEMMA_FINGERPRINT).as_str());
let expected = generation_fingerprint(GEMMA_FINGERPRINT);
let coverage = if dimensions == Some(GEMMA_DIMENSIONS) {
fingerprint
.as_deref()
.and_then(|active| policy_coverage(&expected, active))
} else {
None
};
let prepared_coverage = if revision.is_some() { coverage } else { None };
let policy_matches = match requested_coverage {
SemanticCoverage::TextOnly => prepared_coverage.is_some(),
// Keep the default full-media status's exact configured policy rule.
SemanticCoverage::FullMedia => fingerprint.as_deref() == Some(expected.as_str()),
};
let stale = revision != Some(current) || !policy_matches;
Ok(EmbeddingStatus {
coverage: prepared_coverage,
requested_coverage,
text_indexing_allowed: coverage != Some(SemanticCoverage::FullMedia),
failures,
generation,
chunks,
@@ -374,6 +447,19 @@ impl HybridSearch {
})
}
/// Early text setup never replaces a compatible full generation. The
/// decision is made under the publication build lock, not from UI status.
/// `None` means full-media indexing must resume after its assets are ready.
pub fn index_text_embeddings_preserving_media(&mut self) -> Result<Option<EmbeddingReport>> {
let fingerprint = format!(
"{}:text-only",
generation_fingerprint(self.embedder.fingerprint())
);
self.rebuild_generation_preserving_media(fingerprint, |service, generation, work| {
service.build(generation, work)
})
}
/// Filter each retrieval lane before ranking, preserve filename priority,
/// then fuse keyword and semantic ranks with reciprocal-rank fusion (k=60).
pub fn search(
@@ -827,3 +913,6 @@ mod tests {
Ok(())
}
}
#[cfg(test)]
mod coverage_tests;