Keep prepared search readers warm and verify abrupt indexing recovery

crates/searchi-cli/Cargo.toml

@@ -25,3 +25,4 @@ uuid = { version = "1", features = ["v4"] }
 
 [dev-dependencies]
 tempfile = "3"
+rusqlite = { version = "0.32", features = ["bundled"] }

crates/searchi-cli/tests/crash_recovery.rs

@@ -0,0 +1,297 @@
+//! Abrupt termination of an owned real CLI process tree, not a returned error.
+//! Requires verified local Google assets/decoders. This is not power-loss proof.
+use rusqlite::Connection;
+use serde_json::Value;
+use std::{
+    fs::{self, File},
+    io::Write,
+    path::{Path, PathBuf},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, Instant},
+};
+
+struct OwnedCli {
+    child: Child,
+}
+impl OwnedCli {
+    fn spawn(mut command: Command, stdout: &Path, stderr: &Path) -> Self {
+        #[cfg(unix)]
+        {
+            use std::os::unix::process::CommandExt;
+            // Every worker/decoder inherits this exclusive process group.
+            command.process_group(0);
+        }
+        command
+            .env("SEARCHI_FORCE_CPU", "1")
+            .stdin(Stdio::null())
+            .stdout(File::create(stdout).unwrap())
+            .stderr(File::create(stderr).unwrap());
+        Self {
+            child: command.spawn().unwrap(),
+        }
+    }
+    fn terminate(&mut self) -> std::io::Result<()> {
+        if self.child.try_wait()?.is_some() {
+            return Ok(());
+        }
+        #[cfg(windows)]
+        let status = Command::new("taskkill.exe")
+            .args(["/PID", &self.child.id().to_string(), "/T", "/F"])
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            .status()?;
+        #[cfg(unix)]
+        let status = Command::new("/bin/kill")
+            .args(["-KILL", "--", &format!("-{}", self.child.id())])
+            .status()?;
+        if !status.success() {
+            return Err(std::io::Error::other(
+                "Could not terminate owned CLI process tree",
+            ));
+        }
+        self.child.wait()?;
+        Ok(())
+    }
+    fn finish(&mut self, timeout: Duration, stderr: &Path) {
+        let deadline = Instant::now() + timeout;
+        loop {
+            if let Some(status) = self.child.try_wait().unwrap() {
+                assert!(
+                    status.success(),
+                    "CLI failed: {}",
+                    fs::read_to_string(stderr).unwrap()
+                );
+                return;
+            }
+            assert!(Instant::now() < deadline, "Owned CLI exceeded deadline");
+            thread::sleep(Duration::from_millis(20));
+        }
+    }
+}
+impl Drop for OwnedCli {
+    fn drop(&mut self) {
+        let _ = self.terminate();
+    }
+}
+
+fn cli(db: &Path, command: &str) -> Command {
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_searchi"));
+    cmd.arg(command).arg("--db").arg(db);
+    cmd
+}
+fn run(command: Command, directory: &Path, label: &str) -> Value {
+    let stdout = directory.join(format!("{label}.stdout"));
+    let stderr = directory.join(format!("{label}.stderr"));
+    let mut child = OwnedCli::spawn(command, &stdout, &stderr);
+    child.finish(Duration::from_secs(900), &stderr);
+    serde_json::from_slice(&fs::read(stdout).unwrap()).unwrap()
+}
+fn embed(db: &Path, model: &Path, worker: &Path, ffmpeg: &Path, ffprobe: &Path) -> Command {
+    let mut command = cli(db, "embed");
+    command
+        .arg("--model-dir")
+        .arg(model)
+        .arg("--embedding-worker")
+        .arg(worker)
+        .arg("--cpu-threads")
+        .arg("1")
+        .arg("--ffmpeg")
+        .arg(ffmpeg)
+        .arg("--ffprobe")
+        .arg(ffprobe);
+    command
+}
+fn index(db: &Path, root: &Path, directory: &Path) {
+    let stdout = directory.join("index.stdout");
+    let stderr = directory.join("index.stderr");
+    let mut command = cli(db, "index");
+    command.arg("--root").arg(root);
+    let mut child = OwnedCli::spawn(command, &stdout, &stderr);
+    child.finish(Duration::from_secs(60), &stderr);
+}
+fn required(name: &str) -> PathBuf {
+    PathBuf::from(std::env::var_os(name).unwrap_or_else(|| panic!("{name} required")))
+}
+fn active(db: &Connection) -> i64 {
+    db.query_row(
+        "SELECT active_generation FROM semantic_meta WHERE singleton=1",
+        [],
+        |r| r.get(0),
+    )
+    .unwrap()
+}
+type CommittedChunk = (String, i64, String, Vec<u8>);
+fn completed(db: &Connection, generation: i64) -> Vec<CommittedChunk> {
+    let mut statement = db.prepare("SELECT c.task_key,c.ordinal,c.path,c.vector FROM semantic_chunks c JOIN semantic_tasks t ON t.generation=c.generation AND t.task_key=c.task_key WHERE c.generation=?1 AND t.state='done' ORDER BY c.task_key,c.ordinal").unwrap();
+    statement
+        .query_map([generation], |r| {
+            Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))
+        })
+        .unwrap()
+        .collect::<rusqlite::Result<_>>()
+        .unwrap()
+}
+fn write_audio(path: &Path) {
+    // Standard PCM WAV: three overlapping audio windows under normal limits.
+    let samples = 16_000_u32 * 65;
+    let mut wav = File::create(path).unwrap();
+    for bytes in [
+        b"RIFF".as_slice(),
+        &(36 + samples * 2).to_le_bytes(),
+        b"WAVEfmt ",
+        &16_u32.to_le_bytes(),
+        &1_u16.to_le_bytes(),
+        &1_u16.to_le_bytes(),
+        &16_000_u32.to_le_bytes(),
+        &32_000_u32.to_le_bytes(),
+        &2_u16.to_le_bytes(),
+        &16_u16.to_le_bytes(),
+        b"data",
+        &(samples * 2).to_le_bytes(),
+    ] {
+        wav.write_all(bytes).unwrap();
+    }
+    let mut pcm = Vec::with_capacity(samples as usize * 2);
+    for n in 0..samples {
+        let value = ((n as f64 * 440.0 * std::f64::consts::TAU / 16_000.0).sin() * 4096.0) as i16;
+        pcm.extend(value.to_le_bytes());
+    }
+    wav.write_all(&pcm).unwrap();
+}
+
+#[test]
+#[ignore = "Requires verified Google assets/CPU auto worker and FFmpeg; kills only owned CLI tree"]
+fn real_google_cli_kill_preserves_active_and_resumes_only_complete_groups() {
+    assert_eq!(std::env::var("SEARCHI_FORCE_CPU").unwrap(), "1");
+    let model = required("SEARCHI_GEMMA_MODEL_DIR");
+    let worker = required("SEARCHI_GEMMA_WORKER");
+    let ffmpeg = required("SEARCHI_TEST_FFMPEG");
+    let ffprobe = required("SEARCHI_TEST_FFPROBE");
+    let temp = tempfile::tempdir().unwrap();
+    let root = temp.path().join("sources");
+    fs::create_dir(&root).unwrap();
+    fs::write(
+        root.join("agreement.txt"),
+        "The supplier partnership can end with sixty days written notice.",
+    )
+    .unwrap();
+    let path = temp.path().join("index.db");
+    index(&path, &root, temp.path());
+    run(
+        embed(&path, &model, &worker, &ffmpeg, &ffprobe),
+        temp.path(),
+        "initial",
+    );
+    let db = Connection::open(&path).unwrap();
+    db.busy_timeout(Duration::from_secs(2)).unwrap();
+    let original = active(&db);
+    let old_vectors = completed(&db, original);
+    fs::write(
+        root.join("retry.rs"),
+        "pub fn exponential_backoff(attempt: u32) -> u64 { 100 * 2_u64.pow(attempt.min(8)) }\n",
+    )
+    .unwrap();
+    write_audio(&root.join("tone.wav"));
+    index(&path, &root, temp.path());
+    let stdout = temp.path().join("interrupted.stdout");
+    let stderr = temp.path().join("interrupted.stderr");
+    let mut child = OwnedCli::spawn(
+        embed(&path, &model, &worker, &ffmpeg, &ffprobe),
+        &stdout,
+        &stderr,
+    );
+    let deadline = Instant::now() + Duration::from_secs(300);
+    let generation = loop {
+        assert!(
+            child.child.try_wait().unwrap().is_none(),
+            "CLI finished before interruption boundary: {}",
+            fs::read_to_string(&stderr).unwrap()
+        );
+        let boundary: Option<i64> = db.query_row("SELECT g.id FROM semantic_generations g WHERE g.id!=?1 AND (SELECT count(*) FROM semantic_tasks t WHERE t.generation=g.id AND t.state='done')>=2 AND EXISTS(SELECT 1 FROM semantic_tasks t JOIN semantic_chunks c ON c.generation=t.generation AND c.path=t.path AND c.passage_id=t.passage_id WHERE t.generation=g.id AND t.state='running' AND t.source_type='media' AND c.task_key IS NULL AND c.source_type!='text') LIMIT 1", [original], |r| r.get(0)).ok();
+        if let Some(generation) = boundary {
+            break generation;
+        }
+        assert!(
+            Instant::now() < deadline,
+            "No committed partial media boundary within deadline"
+        );
+        thread::sleep(Duration::from_millis(20));
+    };
+    child.terminate().unwrap();
+    assert_eq!(
+        active(&db),
+        original,
+        "Abrupt kill must not publish staging"
+    );
+    assert_eq!(completed(&db, original), old_vectors);
+    let saved = completed(&db, generation);
+    assert_eq!(saved.len(), 2);
+    let partial: usize = db.query_row("SELECT count(*) FROM semantic_chunks c JOIN semantic_tasks t ON t.generation=c.generation AND t.path=c.path AND t.passage_id=c.passage_id WHERE c.generation=?1 AND t.state='running' AND t.source_type='media' AND c.task_key IS NULL AND c.source_type!='text'",[generation],|r|r.get(0)).unwrap();
+    assert!(
+        partial > 0,
+        "Need actual committed incomplete Google vectors"
+    );
+    let mut old_search = cli(&path, "search");
+    old_search
+        .arg("How can we stop working with the vendor?")
+        .arg("--json")
+        .arg("--model-dir")
+        .arg(&model)
+        .arg("--embedding-worker")
+        .arg(&worker)
+        .arg("--cpu-threads")
+        .arg("1");
+    let previous_results = run(old_search, temp.path(), "old-reader");
+    assert_eq!(previous_results[0]["filename"], "agreement.txt");
+    assert_eq!(
+        active(&db),
+        original,
+        "Searching must not activate unfinished work"
+    );
+    let report = run(
+        embed(&path, &model, &worker, &ffmpeg, &ffprobe),
+        temp.path(),
+        "resumed",
+    );
+    assert_eq!(report["resumed"], true);
+    assert_eq!(report["generation"], generation);
+    assert_eq!(report["document_inference_calls"], 0);
+    assert_eq!(report["reused_sources"], 2);
+    assert_eq!(report["encoded_sources"], 1);
+    assert_eq!(
+        report["audio_inference_calls"], 3,
+        "All incomplete media windows must be encoded afresh"
+    );
+    assert_eq!(report["media_failures"], 0);
+    assert_eq!(active(&db), generation);
+    let after = completed(&db, generation);
+    for chunk in saved {
+        assert!(
+            after.contains(&chunk),
+            "Committed real vector bytes/citations changed"
+        );
+    }
+    let (total,unique):(usize,usize)=db.query_row("SELECT count(*),count(DISTINCT task_key||':'||ordinal) FROM semantic_chunks WHERE generation=?1",[generation],|r|Ok((r.get(0)?,r.get(1)?))).unwrap();
+    assert_eq!(
+        (total, unique),
+        (5, 5),
+        "Partial media chunks must not leak or duplicate"
+    );
+    let mut search = cli(&path, "search");
+    search
+        .arg("retry failed requests with exponential delays")
+        .arg("--code-only")
+        .arg("--json")
+        .arg("--model-dir")
+        .arg(&model)
+        .arg("--embedding-worker")
+        .arg(&worker)
+        .arg("--cpu-threads")
+        .arg("1");
+    let results = run(search, temp.path(), "citation");
+    assert_eq!(results[0]["filename"], "retry.rs");
+    assert_eq!(results[0]["source"]["kind"], "line");
+    assert_eq!(results[0]["source"]["number"], 1);
+    eprintln!("Real CPU CLI crash recovery: generation={generation}, committed_sources=2, discarded_partial_chunks={partial}, resumed_audio_calls=3");
+}

crates/searchi-service/src/lib.rs

@@ -1,6 +1,8 @@
 //! Shared offline hybrid search using real Google embeddings and persisted,
 //! filtered HNSW retrieval. Production-scale validation remains a release gate.
 mod media;
+mod prepared;
+pub use prepared::PreparedGraphs;
 mod pipeline;
 mod routing;
 pub use routing::SemanticOrigin;
@@ -172,6 +174,11 @@ pub struct HybridSearch {
 }
 
 impl HybridSearch {
+    /// Connect background publication to an existing warmed reader.
+    pub fn share_prepared_graphs(&mut self, graphs: &PreparedGraphs) -> Result<()> {
+        self.store.share_prepared_graphs(graphs)
+    }
+
     /// Actual model/runtime/device reported by the persistent native worker.
     pub fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
         self.embedder.worker_info()

crates/searchi-service/src/prepared.rs

@@ -0,0 +1,161 @@
+//! Process-local handoff of immutable graphs prepared by background indexing.
+use anyhow::{ensure, Result};
+use std::{
+    collections::VecDeque,
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex},
+};
+use usearch::Index;
+
+/// Share between services for the same database. No model or query data is shared.
+#[derive(Clone, Default)]
+pub struct PreparedGraphs(Arc<Mutex<State>>);
+#[derive(Default)]
+struct State {
+    identity: Option<(PathBuf, usize)>,
+    entries: VecDeque<Entry>,
+}
+#[derive(Clone)]
+struct Entry {
+    generation: i64,
+    fingerprint: String,
+    checksum: String,
+    graph: Arc<Index>,
+}
+impl PreparedGraphs {
+    pub(crate) fn bind(&self, database: &Path, dimensions: usize) -> Result<()> {
+        let identity = (database.canonicalize()?, dimensions);
+        let mut state = self
+            .0
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Prepared graph registry unavailable"))?;
+        if let Some(existing) = &state.identity {
+            ensure!(
+                *existing == identity,
+                "Prepared graphs belong to another database or dimension"
+            );
+        } else {
+            state.identity = Some(identity);
+        }
+        Ok(())
+    }
+    pub(crate) fn get(
+        &self,
+        generation: i64,
+        fingerprint: &str,
+        checksum: &str,
+    ) -> Result<Option<Arc<Index>>> {
+        let state = self
+            .0
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Prepared graph registry unavailable"))?;
+        Ok(state
+            .entries
+            .iter()
+            .find(|entry| {
+                entry.generation == generation
+                    && entry.fingerprint == fingerprint
+                    && entry.checksum == checksum
+            })
+            .map(|entry| entry.graph.clone()))
+    }
+    pub(crate) fn publish(
+        &self,
+        generation: i64,
+        fingerprint: String,
+        checksum: String,
+        graph: Arc<Index>,
+    ) -> Result<Publication> {
+        let mut state = self
+            .0
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Prepared graph registry unavailable"))?;
+        state.entries.push_back(Entry {
+            generation,
+            fingerprint,
+            checksum,
+            graph: graph.clone(),
+        });
+        while state.entries.len() > 2 {
+            state.entries.pop_front();
+        }
+        Ok(Publication {
+            registry: self.clone(),
+            graph,
+            committed: false,
+        })
+    }
+}
+/// Failed SQL publication removes only this publisher's entry.
+pub(crate) struct Publication {
+    registry: PreparedGraphs,
+    graph: Arc<Index>,
+    committed: bool,
+}
+impl Publication {
+    pub(crate) fn commit(mut self) {
+        self.committed = true;
+    }
+}
+impl Drop for Publication {
+    fn drop(&mut self) {
+        if !self.committed {
+            if let Ok(mut state) = self.registry.0.lock() {
+                state
+                    .entries
+                    .retain(|entry| !Arc::ptr_eq(&entry.graph, &self.graph));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    fn graph() -> Arc<Index> {
+        Arc::new(
+            Index::new(&usearch::IndexOptions {
+                dimensions: 768,
+                ..Default::default()
+            })
+            .unwrap(),
+        )
+    }
+    #[test]
+    fn identity_checksum_and_rollback_are_enforced() -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let database = directory.path().join("index.db");
+        std::fs::write(&database, [])?;
+        let other = directory.path().join("other.db");
+        std::fs::write(&other, [])?;
+        let registry = PreparedGraphs::default();
+        registry.bind(&database, 768)?;
+        assert!(registry.bind(&other, 768).is_err());
+        assert!(registry.bind(&database, 256).is_err());
+        let active = graph();
+        registry
+            .publish(1, "policy".into(), "one".into(), active.clone())?
+            .commit();
+        assert!(registry.get(1, "policy", "wrong")?.is_none());
+        assert!(registry.get(1, "wrong", "one")?.is_none());
+        {
+            let _pending = registry.publish(2, "policy".into(), "two".into(), graph())?;
+            assert!(registry.get(2, "policy", "two")?.is_some());
+        }
+        assert!(registry.get(2, "policy", "two")?.is_none());
+        assert!(Arc::ptr_eq(
+            &active,
+            &registry.get(1, "policy", "one")?.unwrap()
+        ));
+        registry
+            .publish(2, "policy".into(), "two".into(), graph())?
+            .commit();
+        registry
+            .publish(3, "policy".into(), "three".into(), graph())?
+            .commit();
+        assert_eq!(registry.0.lock().unwrap().entries.len(), 2);
+        assert!(registry.get(1, "policy", "one")?.is_none());
+        assert_eq!(active.dimensions(), 768); // Eviction never invalidates a retained reader.
+        Ok(())
+    }
+}

crates/searchi-service/src/store.rs

@@ -45,7 +45,8 @@ pub(crate) struct Store {
     connection: Connection,
     lock_path: PathBuf,
     dimensions: usize,
-    ann_cache: Option<(i64, Index)>,
+    ann_cache: Option<(i64, Arc<Index>)>,
+    prepared_graphs: crate::PreparedGraphs,
     eligibility_cache: Option<EligibilityCache>,
 }
 
@@ -68,6 +69,19 @@ thread_local! { static HASH_CALLS:std::cell::Cell<usize>=const {std::cell::Cell:
 thread_local! { static MEDIA_STAT_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
 
 impl Store {
+    pub(crate) fn share_prepared_graphs(&mut self, graphs: &crate::PreparedGraphs) -> Result<()> {
+        graphs.bind(
+            Path::new(
+                self.connection
+                    .path()
+                    .context("Database path unavailable")?,
+            ),
+            self.dimensions,
+        )?;
+        self.prepared_graphs = graphs.clone();
+        Ok(())
+    }
+
     pub(crate) fn active_fingerprint(&self) -> Result<Option<String>> {
         Ok(self.connection.query_row("SELECT g.fingerprint FROM semantic_meta m JOIN semantic_generations g ON g.id=m.active_generation WHERE m.singleton=1",[],|r|r.get(0)).optional()?)
     }
@@ -170,6 +184,7 @@ impl Store {
             lock_path,
             dimensions,
             ann_cache: None,
+            prepared_graphs: crate::PreparedGraphs::default(),
             eligibility_cache: None,
         })
     }
@@ -328,6 +343,16 @@ impl Store {
         // before acquiring the write transaction. The revision check below
         // still closes races with indexed changes after this validation.
         validate_sources()?;
+        let fingerprint: String = self.connection.query_row(
+            "SELECT fingerprint FROM semantic_generations WHERE id=?1",
+            [generation],
+            |row| row.get(0),
+        )?;
+        let ann = Arc::new(ann);
+        // Readers observing SQL publication already have a prepared immutable graph.
+        let publication =
+            self.prepared_graphs
+                .publish(generation, fingerprint, checksum.clone(), ann.clone())?;
         let transaction = self
             .connection
             .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
@@ -366,6 +391,7 @@ impl Store {
         transaction.execute_batch("UPDATE semantic_meta SET dirty_floor=max(dirty_floor,corpus_revision-100000) WHERE singleton=1;
             DELETE FROM semantic_dirty_paths WHERE revision<(SELECT dirty_floor FROM semantic_meta WHERE singleton=1);")?;
         transaction.commit()?;
+        publication.commit();
         self.ann_cache = Some((generation, ann));
         self.eligibility_cache = None;
         Ok(())
@@ -505,6 +531,23 @@ impl Store {
             return Ok(Vec::new());
         };
         trace.end(generation_span, Default::default());
+        if !self
+            .ann_cache
+            .as_ref()
+            .is_some_and(|(generation, _)| *generation == active)
+        {
+            let prepared = trace.timed("semantic.prepared_graph_lookup", parent, |_, _| {
+                let checksum: String = snapshot.query_row(
+                    "SELECT ann_sha256 FROM semantic_generations WHERE id=?1",
+                    [active],
+                    |row| row.get(0),
+                )?;
+                self.prepared_graphs.get(active, fingerprint, &checksum)
+            })?;
+            if let Some(graph) = prepared {
+                self.ann_cache = Some((active, graph));
+            }
+        }
         let ann_load = trace.begin("semantic.ann_load", parent);
         trace.counter(
             ann_load,
@@ -553,7 +596,7 @@ impl Store {
             ann.reserve_capacity_and_threads(ann.size().max(1), 1)?;
             trace.counter(deserialize_span, "chunks", ann.size() as u64);
             trace.end(deserialize_span, Default::default());
-            self.ann_cache = Some((active, ann));
+            self.ann_cache = Some((active, Arc::new(ann)));
         }
         trace.end(ann_load, Default::default());
         let filter_span = trace.begin("semantic.filter_normalization", parent);
@@ -1417,6 +1460,94 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn failed_sql_activation_removes_prepared_pending_and_preserves_live_reader() -> Result<()> {
+        // Storage publication invariant only; real Google coverage is a separate integration.
+        let directory = tempfile::tempdir()?;
+        let root = directory.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "original relevant content")?;
+        let database = directory.path().join("index.db");
+        let mut lexical = SearchIndex::open(&database)?;
+        lexical.index_folder(&root)?;
+        let passage = lexical.passage_batch(0, 1)?.remove(0);
+        let windows = [TextWindow {
+            text: passage.text.clone(),
+            start_byte: 0,
+            end_byte: passage.text.len(),
+        }];
+        let mut vector = vec![0.0; DIMENSIONS];
+        vector[0] = 1.0;
+        let policy = "test-storage-publication";
+        let graphs = crate::PreparedGraphs::default();
+        let mut writer = Store::open(&database, DIMENSIONS)?;
+        writer.share_prepared_graphs(&graphs)?;
+        let active = writer.begin(policy)?;
+        writer.insert(active, &passage, 0, &windows, &[vector.clone()])?;
+        writer.activate(active)?;
+        let checksum: String = writer.connection.query_row(
+            "SELECT ann_sha256 FROM semantic_generations WHERE id=?1",
+            [active],
+            |r| r.get(0),
+        )?;
+        let mut reader = Store::open(&database, DIMENSIONS)?;
+        reader.share_prepared_graphs(&graphs)?;
+        assert_eq!(
+            reader
+                .search(&vector, policy, 10, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        let retained = reader.ann_cache.as_ref().unwrap().1.clone();
+        let pending = writer.begin(policy)?;
+        writer.insert(pending, &passage, 0, &windows, &[vector.clone()])?;
+        let prepared = writer.build_ann(pending)?;
+        let mut serialized = vec![0; prepared.serialized_length()];
+        prepared.save_to_buffer(&mut serialized)?;
+        let pending_checksum = format!("{:x}", Sha256::digest(&serialized));
+        writer.connection.execute_batch("CREATE TRIGGER reject_activation BEFORE UPDATE OF active_generation ON semantic_meta BEGIN SELECT RAISE(ABORT,'activation rejected'); END")?;
+        assert!(writer
+            .activate(pending)
+            .unwrap_err()
+            .to_string()
+            .contains("activation rejected"));
+        assert_eq!(
+            writer.connection.query_row(
+                "SELECT active_generation FROM semantic_meta WHERE singleton=1",
+                [],
+                |r| r.get::<_, i64>(0)
+            )?,
+            active
+        );
+        assert!(graphs.get(pending, policy, &pending_checksum)?.is_none());
+        assert!(Arc::ptr_eq(
+            &retained,
+            &graphs.get(active, policy, &checksum)?.unwrap()
+        ));
+        assert_eq!(
+            reader
+                .search(&vector, policy, 10, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        assert!(Arc::ptr_eq(
+            &retained,
+            &reader.ann_cache.as_ref().unwrap().1
+        ));
+        writer
+            .connection
+            .execute_batch("DROP TRIGGER reject_activation")?;
+        writer.activate(pending)?;
+        assert_eq!(
+            reader
+                .search(&vector, policy, 10, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        assert_eq!(reader.ann_cache.as_ref().unwrap().0, pending);
+        Ok(())
+    }
+
     #[test]
     fn generation_persists_failed_staging_preserves_active_and_changes_hide_stale_vectors(
     ) -> Result<()> {

crates/searchi-service/tests/prepared_google_cpu.rs

@@ -0,0 +1,87 @@
+//! Genuine Google CPU proof for background prepared-graph publication.
+use anyhow::Result;
+use searchi_core::SearchIndex;
+use searchi_service::{HybridSearch, PreparedGraphs, SearchOptions};
+use std::fs;
+
+#[test]
+#[ignore = "Requires verified Google models and normal worker; CPU only"]
+fn prepared_publication_keeps_warmed_query_reader_hot() -> Result<()> {
+    assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
+    let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Verified models required");
+    let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Normal worker required");
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("documents");
+    fs::create_dir(&root)?;
+    let file = root.join("supplier.txt");
+    fs::write(
+        &file,
+        "The supplier agreement ends with sixty days written notice.",
+    )?;
+    let db = temp.path().join("index.db");
+    let mut lexical = SearchIndex::open(&db)?;
+    lexical.index_folder(&root)?;
+    let graphs = PreparedGraphs::default();
+    let mut indexer = HybridSearch::open(&db, &model, &worker, 1)?;
+    indexer.share_prepared_graphs(&graphs)?;
+    indexer.index_embeddings()?;
+    let mut reader = HybridSearch::open(&db, &model, &worker, 1)?;
+    reader.share_prepared_graphs(&graphs)?;
+    reader.warmup()?;
+    let query = "How can we end our supplier partnership?";
+    reader.search(query, 10, &SearchOptions::default())?;
+    fs::write(
+        &file,
+        "The supplier agreement ends with ninety days written notice.",
+    )?;
+    lexical.index_folder(&root)?;
+    indexer.index_embeddings()?;
+    drop(indexer); // The shared graph outlives the background model worker.
+    let response = reader.search_measured(query, 10, &SearchOptions::default(), true)?;
+    assert_eq!(response.results[0].filename, "supplier.txt");
+    assert!(response.results[0].snippet.contains("ninety days"));
+    let trace = response.metrics.trace.expect("Detailed trace required");
+    assert!(
+        trace
+            .iter()
+            .any(|span| span.counters.get("query_cache_hit") == Some(&1)),
+        "The retained query model must preserve its real-vector LRU across publication"
+    );
+    let loads: Vec<_> = trace
+        .iter()
+        .filter(|span| span.name == "semantic.ann_load")
+        .collect();
+    assert!(!loads.is_empty());
+    assert!(loads
+        .iter()
+        .all(|span| span.counters.get("cache_hit") == Some(&1)));
+    assert!(!trace.iter().any(|span| matches!(
+        span.name.as_str(),
+        "semantic.ann_blob_read" | "semantic.ann_checksum" | "semantic.ann_deserialize"
+    )));
+    let unseen = reader.search_measured(
+        "What notice is required for the contract to end?",
+        10,
+        &SearchOptions::default(),
+        true,
+    )?;
+    assert!(unseen.results[0].snippet.contains("ninety days"));
+    let unseen_trace = unseen.metrics.trace.expect("Unseen query trace required");
+    assert!(
+        unseen_trace
+            .iter()
+            .any(|span| span.counters.get("query_cache_hit") == Some(&0)),
+        "A distinct foreground input must execute genuine query inference"
+    );
+    assert!(unseen_trace
+        .iter()
+        .filter(|span| span.name == "semantic.ann_load")
+        .all(|span| span.counters.get("cache_hit") == Some(&1)));
+    assert!(!unseen_trace.iter().any(|span| matches!(
+        span.name.as_str(),
+        "semantic.ann_blob_read" | "semantic.ann_checksum" | "semantic.ann_deserialize"
+    )));
+    eprintln!("Prepared reader CPU fixture: retained_query_ms={:.3}, unseen_query_ms={:.3}; one document, not a scale benchmark",
+        response.metrics.total_ms, unseen.metrics.total_ms);
+    Ok(())
+}