Reuse completed Google embeddings and resume semantic staging

crates/searchi-service/src/store.rs

@@ -1,3 +1,5 @@
+mod incremental;
+pub(crate) use incremental::SourceTask;
 use std::{
     collections::{HashMap, HashSet},
     fs::{File, OpenOptions},
@@ -47,7 +49,7 @@ pub(crate) struct Store {
     eligibility_cache: Option<EligibilityCache>,
 }
 
-pub(crate) struct BuildLock(File);
+pub(crate) struct BuildLock(File, PathBuf);
 impl Drop for BuildLock {
     fn drop(&mut self) {
         let _ = FileExt::unlock(&self.0);
@@ -95,7 +97,7 @@ impl Store {
             |row| row.get(0),
         )?;
         ensure!(
-            (1..=6).contains(&version),
+            (1..=7).contains(&version),
             "Unsupported semantic schema version {version}"
         );
         initialization.execute_batch("
@@ -159,6 +161,7 @@ impl Store {
             }
         }
         initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
+        incremental::migrate(&initialization)?;
         initialization.commit()?;
         let canonical = db.canonicalize()?;
         let lock_path = PathBuf::from(format!("{}.semantic.lock", canonical.to_string_lossy()));
@@ -180,9 +183,10 @@ impl Store {
             .open(&self.lock_path)?;
         file.try_lock_exclusive()
             .context("Another semantic rebuild is running")?;
-        Ok(BuildLock(file))
+        Ok(BuildLock(file, self.lock_path.clone()))
     }
 
+    #[cfg(test)]
     pub(crate) fn begin(&mut self, fingerprint: &str) -> Result<i64> {
         let transaction = self.connection.transaction()?;
         // Only called while holding the cross-process lock: abandoned staging
@@ -299,7 +303,12 @@ impl Store {
         Ok(())
     }
 
+    #[cfg(test)]
     pub(crate) fn activate(&mut self, generation: i64) -> Result<()> {
+        self.activate_revision(generation, None)
+    }
+
+    fn activate_revision(&mut self, generation: i64, revision: Option<i64>) -> Result<()> {
         // A failed graph build or serialization leaves the old generation active.
         let ann = self.build_ann(generation)?;
         let length = ann.serialized_length();
@@ -310,7 +319,21 @@ impl Store {
         let mut serialized = vec![0_u8; length];
         ann.save_to_buffer(&mut serialized)?;
         let checksum = format!("{:x}", Sha256::digest(&serialized));
-        let transaction = self.connection.transaction()?;
+        let transaction = self
+            .connection
+            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        if let Some(expected) = revision {
+            let current: i64 = transaction.query_row(
+                "SELECT corpus_revision FROM semantic_meta WHERE singleton=1",
+                [],
+                |r| r.get(0),
+            )?;
+            ensure!(
+                current == expected,
+                "Indexed corpus changed during semantic rebuild; retry required"
+            );
+            incremental::validate_completion(&transaction, generation)?;
+        }
         ensure!(
             transaction.query_row(
                 "SELECT EXISTS(SELECT 1 FROM semantic_generations WHERE id=?1)",
@@ -389,6 +412,7 @@ impl Store {
         Ok(ann)
     }
 
+    #[cfg(test)]
     pub(crate) fn abandon(&mut self, generation: i64) -> Result<()> {
         self.connection.execute("DELETE FROM semantic_generations WHERE id=?1 AND id!=COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1)", [generation])?;
         Ok(())