Keep prepared search readers warm and verify abrupt indexing recovery

crates/searchi-service/src/lib.rs

@@ -1,6 +1,8 @@
 //! Shared offline hybrid search using real Google embeddings and persisted,
 //! filtered HNSW retrieval. Production-scale validation remains a release gate.
 mod media;
+mod prepared;
+pub use prepared::PreparedGraphs;
 mod pipeline;
 mod routing;
 pub use routing::SemanticOrigin;
@@ -172,6 +174,11 @@ pub struct HybridSearch {
 }
 
 impl HybridSearch {
+    /// Connect background publication to an existing warmed reader.
+    pub fn share_prepared_graphs(&mut self, graphs: &PreparedGraphs) -> Result<()> {
+        self.store.share_prepared_graphs(graphs)
+    }
+
     /// Actual model/runtime/device reported by the persistent native worker.
     pub fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
         self.embedder.worker_info()

crates/searchi-service/src/prepared.rs

@@ -0,0 +1,161 @@
+//! Process-local handoff of immutable graphs prepared by background indexing.
+use anyhow::{ensure, Result};
+use std::{
+    collections::VecDeque,
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex},
+};
+use usearch::Index;
+
+/// Share between services for the same database. No model or query data is shared.
+#[derive(Clone, Default)]
+pub struct PreparedGraphs(Arc<Mutex<State>>);
+#[derive(Default)]
+struct State {
+    identity: Option<(PathBuf, usize)>,
+    entries: VecDeque<Entry>,
+}
+#[derive(Clone)]
+struct Entry {
+    generation: i64,
+    fingerprint: String,
+    checksum: String,
+    graph: Arc<Index>,
+}
+impl PreparedGraphs {
+    pub(crate) fn bind(&self, database: &Path, dimensions: usize) -> Result<()> {
+        let identity = (database.canonicalize()?, dimensions);
+        let mut state = self
+            .0
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Prepared graph registry unavailable"))?;
+        if let Some(existing) = &state.identity {
+            ensure!(
+                *existing == identity,
+                "Prepared graphs belong to another database or dimension"
+            );
+        } else {
+            state.identity = Some(identity);
+        }
+        Ok(())
+    }
+    pub(crate) fn get(
+        &self,
+        generation: i64,
+        fingerprint: &str,
+        checksum: &str,
+    ) -> Result<Option<Arc<Index>>> {
+        let state = self
+            .0
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Prepared graph registry unavailable"))?;
+        Ok(state
+            .entries
+            .iter()
+            .find(|entry| {
+                entry.generation == generation
+                    && entry.fingerprint == fingerprint
+                    && entry.checksum == checksum
+            })
+            .map(|entry| entry.graph.clone()))
+    }
+    pub(crate) fn publish(
+        &self,
+        generation: i64,
+        fingerprint: String,
+        checksum: String,
+        graph: Arc<Index>,
+    ) -> Result<Publication> {
+        let mut state = self
+            .0
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Prepared graph registry unavailable"))?;
+        state.entries.push_back(Entry {
+            generation,
+            fingerprint,
+            checksum,
+            graph: graph.clone(),
+        });
+        while state.entries.len() > 2 {
+            state.entries.pop_front();
+        }
+        Ok(Publication {
+            registry: self.clone(),
+            graph,
+            committed: false,
+        })
+    }
+}
+/// Failed SQL publication removes only this publisher's entry.
+pub(crate) struct Publication {
+    registry: PreparedGraphs,
+    graph: Arc<Index>,
+    committed: bool,
+}
+impl Publication {
+    pub(crate) fn commit(mut self) {
+        self.committed = true;
+    }
+}
+impl Drop for Publication {
+    fn drop(&mut self) {
+        if !self.committed {
+            if let Ok(mut state) = self.registry.0.lock() {
+                state
+                    .entries
+                    .retain(|entry| !Arc::ptr_eq(&entry.graph, &self.graph));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    fn graph() -> Arc<Index> {
+        Arc::new(
+            Index::new(&usearch::IndexOptions {
+                dimensions: 768,
+                ..Default::default()
+            })
+            .unwrap(),
+        )
+    }
+    #[test]
+    fn identity_checksum_and_rollback_are_enforced() -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let database = directory.path().join("index.db");
+        std::fs::write(&database, [])?;
+        let other = directory.path().join("other.db");
+        std::fs::write(&other, [])?;
+        let registry = PreparedGraphs::default();
+        registry.bind(&database, 768)?;
+        assert!(registry.bind(&other, 768).is_err());
+        assert!(registry.bind(&database, 256).is_err());
+        let active = graph();
+        registry
+            .publish(1, "policy".into(), "one".into(), active.clone())?
+            .commit();
+        assert!(registry.get(1, "policy", "wrong")?.is_none());
+        assert!(registry.get(1, "wrong", "one")?.is_none());
+        {
+            let _pending = registry.publish(2, "policy".into(), "two".into(), graph())?;
+            assert!(registry.get(2, "policy", "two")?.is_some());
+        }
+        assert!(registry.get(2, "policy", "two")?.is_none());
+        assert!(Arc::ptr_eq(
+            &active,
+            &registry.get(1, "policy", "one")?.unwrap()
+        ));
+        registry
+            .publish(2, "policy".into(), "two".into(), graph())?
+            .commit();
+        registry
+            .publish(3, "policy".into(), "three".into(), graph())?
+            .commit();
+        assert_eq!(registry.0.lock().unwrap().entries.len(), 2);
+        assert!(registry.get(1, "policy", "one")?.is_none());
+        assert_eq!(active.dimensions(), 768); // Eviction never invalidates a retained reader.
+        Ok(())
+    }
+}

crates/searchi-service/src/store.rs

@@ -45,7 +45,8 @@ pub(crate) struct Store {
     connection: Connection,
     lock_path: PathBuf,
     dimensions: usize,
-    ann_cache: Option<(i64, Index)>,
+    ann_cache: Option<(i64, Arc<Index>)>,
+    prepared_graphs: crate::PreparedGraphs,
     eligibility_cache: Option<EligibilityCache>,
 }
 
@@ -68,6 +69,19 @@ thread_local! { static HASH_CALLS:std::cell::Cell<usize>=const {std::cell::Cell:
 thread_local! { static MEDIA_STAT_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
 
 impl Store {
+    pub(crate) fn share_prepared_graphs(&mut self, graphs: &crate::PreparedGraphs) -> Result<()> {
+        graphs.bind(
+            Path::new(
+                self.connection
+                    .path()
+                    .context("Database path unavailable")?,
+            ),
+            self.dimensions,
+        )?;
+        self.prepared_graphs = graphs.clone();
+        Ok(())
+    }
+
     pub(crate) fn active_fingerprint(&self) -> Result<Option<String>> {
         Ok(self.connection.query_row("SELECT g.fingerprint FROM semantic_meta m JOIN semantic_generations g ON g.id=m.active_generation WHERE m.singleton=1",[],|r|r.get(0)).optional()?)
     }
@@ -170,6 +184,7 @@ impl Store {
             lock_path,
             dimensions,
             ann_cache: None,
+            prepared_graphs: crate::PreparedGraphs::default(),
             eligibility_cache: None,
         })
     }
@@ -328,6 +343,16 @@ impl Store {
         // before acquiring the write transaction. The revision check below
         // still closes races with indexed changes after this validation.
         validate_sources()?;
+        let fingerprint: String = self.connection.query_row(
+            "SELECT fingerprint FROM semantic_generations WHERE id=?1",
+            [generation],
+            |row| row.get(0),
+        )?;
+        let ann = Arc::new(ann);
+        // Readers observing SQL publication already have a prepared immutable graph.
+        let publication =
+            self.prepared_graphs
+                .publish(generation, fingerprint, checksum.clone(), ann.clone())?;
         let transaction = self
             .connection
             .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
@@ -366,6 +391,7 @@ impl Store {
         transaction.execute_batch("UPDATE semantic_meta SET dirty_floor=max(dirty_floor,corpus_revision-100000) WHERE singleton=1;
             DELETE FROM semantic_dirty_paths WHERE revision<(SELECT dirty_floor FROM semantic_meta WHERE singleton=1);")?;
         transaction.commit()?;
+        publication.commit();
         self.ann_cache = Some((generation, ann));
         self.eligibility_cache = None;
         Ok(())
@@ -505,6 +531,23 @@ impl Store {
             return Ok(Vec::new());
         };
         trace.end(generation_span, Default::default());
+        if !self
+            .ann_cache
+            .as_ref()
+            .is_some_and(|(generation, _)| *generation == active)
+        {
+            let prepared = trace.timed("semantic.prepared_graph_lookup", parent, |_, _| {
+                let checksum: String = snapshot.query_row(
+                    "SELECT ann_sha256 FROM semantic_generations WHERE id=?1",
+                    [active],
+                    |row| row.get(0),
+                )?;
+                self.prepared_graphs.get(active, fingerprint, &checksum)
+            })?;
+            if let Some(graph) = prepared {
+                self.ann_cache = Some((active, graph));
+            }
+        }
         let ann_load = trace.begin("semantic.ann_load", parent);
         trace.counter(
             ann_load,
@@ -553,7 +596,7 @@ impl Store {
             ann.reserve_capacity_and_threads(ann.size().max(1), 1)?;
             trace.counter(deserialize_span, "chunks", ann.size() as u64);
             trace.end(deserialize_span, Default::default());
-            self.ann_cache = Some((active, ann));
+            self.ann_cache = Some((active, Arc::new(ann)));
         }
         trace.end(ann_load, Default::default());
         let filter_span = trace.begin("semantic.filter_normalization", parent);
@@ -1417,6 +1460,94 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn failed_sql_activation_removes_prepared_pending_and_preserves_live_reader() -> Result<()> {
+        // Storage publication invariant only; real Google coverage is a separate integration.
+        let directory = tempfile::tempdir()?;
+        let root = directory.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "original relevant content")?;
+        let database = directory.path().join("index.db");
+        let mut lexical = SearchIndex::open(&database)?;
+        lexical.index_folder(&root)?;
+        let passage = lexical.passage_batch(0, 1)?.remove(0);
+        let windows = [TextWindow {
+            text: passage.text.clone(),
+            start_byte: 0,
+            end_byte: passage.text.len(),
+        }];
+        let mut vector = vec![0.0; DIMENSIONS];
+        vector[0] = 1.0;
+        let policy = "test-storage-publication";
+        let graphs = crate::PreparedGraphs::default();
+        let mut writer = Store::open(&database, DIMENSIONS)?;
+        writer.share_prepared_graphs(&graphs)?;
+        let active = writer.begin(policy)?;
+        writer.insert(active, &passage, 0, &windows, &[vector.clone()])?;
+        writer.activate(active)?;
+        let checksum: String = writer.connection.query_row(
+            "SELECT ann_sha256 FROM semantic_generations WHERE id=?1",
+            [active],
+            |r| r.get(0),
+        )?;
+        let mut reader = Store::open(&database, DIMENSIONS)?;
+        reader.share_prepared_graphs(&graphs)?;
+        assert_eq!(
+            reader
+                .search(&vector, policy, 10, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        let retained = reader.ann_cache.as_ref().unwrap().1.clone();
+        let pending = writer.begin(policy)?;
+        writer.insert(pending, &passage, 0, &windows, &[vector.clone()])?;
+        let prepared = writer.build_ann(pending)?;
+        let mut serialized = vec![0; prepared.serialized_length()];
+        prepared.save_to_buffer(&mut serialized)?;
+        let pending_checksum = format!("{:x}", Sha256::digest(&serialized));
+        writer.connection.execute_batch("CREATE TRIGGER reject_activation BEFORE UPDATE OF active_generation ON semantic_meta BEGIN SELECT RAISE(ABORT,'activation rejected'); END")?;
+        assert!(writer
+            .activate(pending)
+            .unwrap_err()
+            .to_string()
+            .contains("activation rejected"));
+        assert_eq!(
+            writer.connection.query_row(
+                "SELECT active_generation FROM semantic_meta WHERE singleton=1",
+                [],
+                |r| r.get::<_, i64>(0)
+            )?,
+            active
+        );
+        assert!(graphs.get(pending, policy, &pending_checksum)?.is_none());
+        assert!(Arc::ptr_eq(
+            &retained,
+            &graphs.get(active, policy, &checksum)?.unwrap()
+        ));
+        assert_eq!(
+            reader
+                .search(&vector, policy, 10, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        assert!(Arc::ptr_eq(
+            &retained,
+            &reader.ann_cache.as_ref().unwrap().1
+        ));
+        writer
+            .connection
+            .execute_batch("DROP TRIGGER reject_activation")?;
+        writer.activate(pending)?;
+        assert_eq!(
+            reader
+                .search(&vector, policy, 10, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        assert_eq!(reader.ann_cache.as_ref().unwrap().0, pending);
+        Ok(())
+    }
+
     #[test]
     fn generation_persists_failed_staging_preserves_active_and_changes_hide_stale_vectors(
     ) -> Result<()> {

crates/searchi-service/tests/prepared_google_cpu.rs

@@ -0,0 +1,87 @@
+//! Genuine Google CPU proof for background prepared-graph publication.
+use anyhow::Result;
+use searchi_core::SearchIndex;
+use searchi_service::{HybridSearch, PreparedGraphs, SearchOptions};
+use std::fs;
+
+#[test]
+#[ignore = "Requires verified Google models and normal worker; CPU only"]
+fn prepared_publication_keeps_warmed_query_reader_hot() -> Result<()> {
+    assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
+    let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Verified models required");
+    let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Normal worker required");
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("documents");
+    fs::create_dir(&root)?;
+    let file = root.join("supplier.txt");
+    fs::write(
+        &file,
+        "The supplier agreement ends with sixty days written notice.",
+    )?;
+    let db = temp.path().join("index.db");
+    let mut lexical = SearchIndex::open(&db)?;
+    lexical.index_folder(&root)?;
+    let graphs = PreparedGraphs::default();
+    let mut indexer = HybridSearch::open(&db, &model, &worker, 1)?;
+    indexer.share_prepared_graphs(&graphs)?;
+    indexer.index_embeddings()?;
+    let mut reader = HybridSearch::open(&db, &model, &worker, 1)?;
+    reader.share_prepared_graphs(&graphs)?;
+    reader.warmup()?;
+    let query = "How can we end our supplier partnership?";
+    reader.search(query, 10, &SearchOptions::default())?;
+    fs::write(
+        &file,
+        "The supplier agreement ends with ninety days written notice.",
+    )?;
+    lexical.index_folder(&root)?;
+    indexer.index_embeddings()?;
+    drop(indexer); // The shared graph outlives the background model worker.
+    let response = reader.search_measured(query, 10, &SearchOptions::default(), true)?;
+    assert_eq!(response.results[0].filename, "supplier.txt");
+    assert!(response.results[0].snippet.contains("ninety days"));
+    let trace = response.metrics.trace.expect("Detailed trace required");
+    assert!(
+        trace
+            .iter()
+            .any(|span| span.counters.get("query_cache_hit") == Some(&1)),
+        "The retained query model must preserve its real-vector LRU across publication"
+    );
+    let loads: Vec<_> = trace
+        .iter()
+        .filter(|span| span.name == "semantic.ann_load")
+        .collect();
+    assert!(!loads.is_empty());
+    assert!(loads
+        .iter()
+        .all(|span| span.counters.get("cache_hit") == Some(&1)));
+    assert!(!trace.iter().any(|span| matches!(
+        span.name.as_str(),
+        "semantic.ann_blob_read" | "semantic.ann_checksum" | "semantic.ann_deserialize"
+    )));
+    let unseen = reader.search_measured(
+        "What notice is required for the contract to end?",
+        10,
+        &SearchOptions::default(),
+        true,
+    )?;
+    assert!(unseen.results[0].snippet.contains("ninety days"));
+    let unseen_trace = unseen.metrics.trace.expect("Unseen query trace required");
+    assert!(
+        unseen_trace
+            .iter()
+            .any(|span| span.counters.get("query_cache_hit") == Some(&0)),
+        "A distinct foreground input must execute genuine query inference"
+    );
+    assert!(unseen_trace
+        .iter()
+        .filter(|span| span.name == "semantic.ann_load")
+        .all(|span| span.counters.get("cache_hit") == Some(&1)));
+    assert!(!unseen_trace.iter().any(|span| matches!(
+        span.name.as_str(),
+        "semantic.ann_blob_read" | "semantic.ann_checksum" | "semantic.ann_deserialize"
+    )));
+    eprintln!("Prepared reader CPU fixture: retained_query_ms={:.3}, unseen_query_ms={:.3}; one document, not a scale benchmark",
+        response.metrics.total_ms, unseen.metrics.total_ms);
+    Ok(())
+}