Reuse completed Google embeddings and resume semantic staging

crates/searchi-service/src/lib.rs

@@ -1,6 +1,7 @@
 //! Shared offline hybrid search using real Google embeddings and persisted,
 //! filtered HNSW retrieval. Production-scale validation remains a release gate.
 mod media;
+mod pipeline;
 mod routing;
 pub use routing::SemanticOrigin;
 mod store;
@@ -131,6 +132,14 @@ pub struct EmbeddingReport {
     pub chunks: usize,
     pub generation: i64,
     pub model_fingerprint: String,
+    /// Completed groups copied or resumed without document/media inference.
+    pub reused_sources: usize,
+    pub reused_chunks: usize,
+    pub encoded_sources: usize,
+    pub document_inference_calls: usize,
+    pub image_inference_calls: usize,
+    pub audio_inference_calls: usize,
+    pub resumed: bool,
 }
 
 #[derive(Debug, Serialize)]
@@ -206,7 +215,7 @@ impl HybridSearch {
             });
         }
         ensure!(
-            version == 6,
+            (6..=7).contains(&version),
             "Unsupported semantic schema version {version}"
         );
         let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
@@ -285,88 +294,16 @@ impl HybridSearch {
         Ok(())
     }
 
-    /// Build a replacement generation in bounded CPU batches. Inference or
-    /// storage errors leave the previous generation active. A crashed build is
-    /// reclaimed on the next explicit rebuild under an OS advisory lock.
+    /// Build or resume a replacement generation in bounded batches. Completed
+    /// groups can be reused; failures retain staging work and the ready reader.
     pub fn index_embeddings(&mut self) -> Result<EmbeddingReport> {
         let fingerprint = format!(
             "{}:text-only",
             generation_fingerprint(self.embedder.fingerprint())
         );
-        self.rebuild_generation(fingerprint, Self::build)
-    }
-
-    /// Keep the reader's policy aligned with the active generation on every
-    /// failure boundary, including lock admission, activation and cleanup.
-    fn rebuild_generation(
-        &mut self,
-        fingerprint: String,
-        build: impl FnOnce(&mut Self, i64) -> Result<EmbeddingReport>,
-    ) -> Result<EmbeddingReport> {
-        let _lock = self.store.build_lock()?;
-        let generation = self.store.begin(&fingerprint)?;
-        let previous_fingerprint = std::mem::replace(&mut self.generation_fingerprint, fingerprint);
-        let built = build(self, generation).and_then(|report| {
-            self.store.activate(generation)?;
-            Ok(report)
-        });
-        match built {
-            Ok(report) => Ok(report),
-            Err(error) => {
-                self.generation_fingerprint = previous_fingerprint;
-                if let Err(cleanup) = self.store.abandon(generation) {
-                    return Err(error.context(format!(
-                        "Failed to discard staging generation {generation}: {cleanup:#}"
-                    )));
-                }
-                Err(error)
-            }
-        }
-    }
-
-    fn build(&mut self, generation: i64) -> Result<EmbeddingReport> {
-        let mut report = EmbeddingReport {
-            media_files: 0,
-            media_failures: 0,
-            failures: Vec::new(),
-            passages: 0,
-            chunks: 0,
-            generation,
-            model_fingerprint: self.generation_fingerprint.clone(),
-        };
-        let mut after_id = 0;
-        loop {
-            let passages = self.lexical.passage_batch(after_id, 64)?;
-            if passages.is_empty() {
-                break;
-            }
-            for passage in passages {
-                after_id = passage.id;
-                if passage.text.trim().is_empty() {
-                    continue;
-                }
-                let title = bounded_title(&passage.filename);
-                let windows = passage_windows(&mut *self.embedder, &passage.text, title)?;
-                ensure!(
-                    report.chunks + windows.len() <= MAX_SCAN_CHUNKS,
-                    "Corpus exceeds ANN generation resource budget of {MAX_SCAN_CHUNKS} chunks"
-                );
-                for (batch_index, batch) in windows.chunks(MAX_BATCH).enumerate() {
-                    let texts: Vec<_> = batch.iter().map(|window| window.text.as_str()).collect();
-                    let vectors = self.embedder.embed_documents(&texts, title)?;
-                    self.store.insert(
-                        generation,
-                        &passage,
-                        batch_index * MAX_BATCH,
-                        batch,
-                        &vectors,
-                    )?;
-                }
-                report.chunks += windows.len();
-                report.passages += 1;
-            }
-        }
-        Ok(report)
+        self.rebuild_generation(fingerprint, |service, generation, work| {
+            service.build(generation, work)
+        })
     }
 
     /// Filter each retrieval lane before ranking, preserve filename priority,