Prepare text search during media setup and retain query sessions through promotion
This commit is contained in:
1 parent
af6ce29125
commit
9aae3f8216
15 files changed
+1235
-65
No files matched your search
@@ -44,7 +44,7 @@ pub(super) fn text_task(passage: &IndexedPassage, policy: &str) -> Result<Source
|
||||
source_type: "text".into(),
|
||||
source_hash: content_hash(&passage.text),
|
||||
title_hash: content_hash(bounded_title(&passage.filename)),
|
||||
policy: policy.into(),
|
||||
policy: policy_base(policy).into(),
|
||||
passage_id: passage.id,
|
||||
})
|
||||
}
|
||||
@@ -56,7 +56,29 @@ impl HybridSearch {
|
||||
media_limits: Option<&searchi_media::MediaLimits>,
|
||||
build: impl FnOnce(&mut Self, i64, &mut BuildWork) -> Result<()>,
|
||||
) -> Result<EmbeddingReport> {
|
||||
self.rebuild_generation_guarded(fingerprint, media_limits, false, build)?
|
||||
.ok_or_else(|| anyhow::anyhow!("Explicit semantic rebuild was unexpectedly deferred"))
|
||||
}
|
||||
|
||||
pub(super) fn rebuild_generation_preserving_media(
|
||||
&mut self,
|
||||
fingerprint: String,
|
||||
build: impl FnOnce(&mut Self, i64, &mut BuildWork) -> Result<()>,
|
||||
) -> Result<Option<EmbeddingReport>> {
|
||||
self.rebuild_generation_guarded(fingerprint, None, true, build)
|
||||
}
|
||||
|
||||
fn rebuild_generation_guarded(
|
||||
&mut self,
|
||||
fingerprint: String,
|
||||
media_limits: Option<&searchi_media::MediaLimits>,
|
||||
preserve_full_media: bool,
|
||||
build: impl FnOnce(&mut Self, i64, &mut BuildWork) -> Result<()>,
|
||||
) -> Result<Option<EmbeddingReport>> {
|
||||
let lock = self.store.build_lock()?;
|
||||
if preserve_full_media && self.store.has_compatible_full_generation(&fingerprint)? {
|
||||
return Ok(None);
|
||||
}
|
||||
let (generation, resumed) = self.store.start_or_resume(&fingerprint, &lock)?;
|
||||
// This revision covers the entire discovery/build/finalization pass.
|
||||
// A concurrent lexical update aborts publication; next invocation
|
||||
@@ -109,7 +131,7 @@ impl HybridSearch {
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(work.report)
|
||||
Ok(Some(work.report))
|
||||
});
|
||||
if result.is_err() {
|
||||
self.generation_fingerprint = previous_fingerprint;
|
||||
@@ -356,4 +378,41 @@ mod tests {
|
||||
assert_eq!(staging, 1, "Resume must retain the same staging generation");
|
||||
Ok(())
|
||||
}
|
||||
#[test]
|
||||
fn discovery_and_encoding_use_the_same_normalized_text_component_identity() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let root = temp.path().join("docs");
|
||||
std::fs::create_dir(&root)?;
|
||||
std::fs::write(
|
||||
root.join("notes.txt"),
|
||||
"Exact source bytes before a deliberately failed encoder.",
|
||||
)?;
|
||||
let db = temp.path().join("index.db");
|
||||
SearchIndex::open(&db)?.index_folder(&root)?;
|
||||
let mut service = HybridSearch::with_provider(&db, Box::new(RejectInference))?;
|
||||
let base = generation_fingerprint(service.embedder.fingerprint());
|
||||
service.generation_fingerprint = format!("{base}:text-only");
|
||||
let discovered = service.discover_tasks(None)?.remove(0);
|
||||
assert_eq!(discovered.policy, base);
|
||||
let error = service
|
||||
.index_text_embeddings_preserving_media()
|
||||
.unwrap_err();
|
||||
assert!(format!("{error:#}").contains("injected primary inference failure"));
|
||||
let connection = rusqlite::Connection::open(&db)?;
|
||||
let encoded: (String, String, String, String) = connection.query_row(
|
||||
"SELECT task_key,source_hash,title_hash,policy FROM semantic_tasks",
|
||||
[],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
|
||||
)?;
|
||||
assert_eq!(
|
||||
encoded,
|
||||
(
|
||||
discovered.key,
|
||||
discovered.source_hash,
|
||||
discovered.title_hash,
|
||||
discovered.policy
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user