Verify extraction provenance and prune semantic staging before inference

crates/searchi-service/tests/publication_google_cpu.rs

@@ -0,0 +1,189 @@
+//! Publication faults exercised with the actual Google CPU encoder.
+use anyhow::Result;
+use rusqlite::Connection;
+use searchi_core::SearchIndex;
+use searchi_semantic::{GemmaEmbedder, TextWindow};
+use searchi_service::{EmbeddingProvider, HybridSearch, SearchOptions};
+use std::{
+    fs,
+    path::PathBuf,
+    sync::{Arc, Mutex},
+};
+
+#[derive(Default)]
+struct Control {
+    calls: usize,
+    mutation: Option<(PathBuf, String)>,
+}
+struct GoogleWithMutation {
+    google: GemmaEmbedder,
+    control: Arc<Mutex<Control>>,
+}
+impl EmbeddingProvider for GoogleWithMutation {
+    fn fingerprint(&self) -> &str {
+        EmbeddingProvider::fingerprint(&self.google)
+    }
+    fn dimensions(&self) -> usize {
+        EmbeddingProvider::dimensions(&self.google)
+    }
+    fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>> {
+        self.google.split_windows(text, title)
+    }
+    fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result<Vec<Vec<f32>>> {
+        let vectors = self.google.embed_documents(texts, title)?;
+        let mut control = self.control.lock().unwrap();
+        control.calls += texts.len();
+        if let Some((path, replacement)) = control.mutation.take() {
+            let metadata = fs::metadata(&path)?;
+            assert_eq!(metadata.len(), replacement.len() as u64);
+            fs::write(&path, replacement)?;
+            fs::File::options()
+                .write(true)
+                .open(&path)?
+                .set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?;
+            assert_eq!(fs::metadata(&path)?.modified()?, metadata.modified()?);
+        }
+        Ok(vectors)
+    }
+    fn embed_query(&mut self, text: &str) -> Result<Vec<f32>> {
+        self.google.embed_query(text)
+    }
+    fn embed_code_query(&mut self, text: &str) -> Result<Vec<f32>> {
+        self.google.embed_code_query(text)
+    }
+}
+
+#[test]
+#[ignore = "Requires verified Google models and normal native worker; CPU only"]
+fn genuine_google_same_metadata_mutation_rejects_publication_and_preserves_ready_sources(
+) -> Result<()> {
+    assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
+    let google = GemmaEmbedder::load(
+        std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Verified Google assets required"),
+        std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Normal auto worker required"),
+        1,
+    )?;
+    assert_eq!(google.info.device, "cpu");
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("documents");
+    fs::create_dir(&root)?;
+    let stable = root.join("garden.txt");
+    fs::write(
+        &stable,
+        "The garden irrigation waters tomato plants every morning.",
+    )?;
+    let changing = root.join("supplier.txt");
+    fs::write(
+        &changing,
+        "The supplier contract ends after sixty days written notice.",
+    )?;
+    let db = temp.path().join("index.db");
+    let mut lexical = SearchIndex::open(&db)?;
+    lexical.index_folder(&root)?;
+    let control = Arc::new(Mutex::new(Control::default()));
+    let mut service = HybridSearch::with_provider(
+        &db,
+        Box::new(GoogleWithMutation {
+            google,
+            control: control.clone(),
+        }),
+    )?;
+    service.index_embeddings()?;
+    let connection = Connection::open(&db)?;
+    let active = || {
+        connection.query_row(
+            "SELECT active_generation FROM semantic_meta WHERE singleton=1",
+            [],
+            |r| r.get::<_, i64>(0),
+        )
+    };
+    let first = active()?;
+    let stable_vector: Vec<u8> = connection.query_row(
+        "SELECT vector FROM semantic_chunks WHERE generation=?1 AND path LIKE '%garden.txt'",
+        [first],
+        |r| r.get(0),
+    )?;
+    fs::write(
+        &changing,
+        "The supplier contract ends after ninety days written notice.",
+    )?;
+    lexical.index_folder(&root)?;
+    control.lock().unwrap().mutation = Some((
+        changing.clone(),
+        "The supplier contract ends after thirty days written notice.".into(),
+    ));
+    let error = service.index_embeddings().unwrap_err();
+    assert!(
+        format!("{error:#}").contains("Source changed during semantic indexing"),
+        "{error:#}"
+    );
+    assert_eq!(
+        active()?,
+        first,
+        "Changed bytes must never publish a stale generation"
+    );
+    assert_eq!(control.lock().unwrap().calls, 3);
+    let options = SearchOptions::default();
+    let results = service.search_semantic("tomato irrigation", 10, &options)?;
+    assert!(results
+        .iter()
+        .any(|r| r.filename == "garden.txt" && r.source.is_some()));
+
+    // A normal rescan must detect changed parser input even though the
+    // mutation restored size and modification time; no metadata workaround.
+    lexical.index_folder(&root)?;
+    let report = service.index_embeddings()?;
+    assert!(report.resumed);
+    assert_eq!(report.reused_sources, 1);
+    assert_eq!(report.encoded_sources, 1);
+    assert_eq!(control.lock().unwrap().calls, 4);
+    let latest = active()?;
+    assert_ne!(latest, first);
+    let latest_stable: Vec<u8> = connection.query_row(
+        "SELECT vector FROM semantic_chunks WHERE generation=?1 AND path LIKE '%garden.txt'",
+        [latest],
+        |r| r.get(0),
+    )?;
+    assert_eq!(latest_stable, stable_vector);
+    let results = service.search_semantic("supplier contract notice", 10, &options)?;
+    let supplier = results
+        .iter()
+        .find(|r| r.filename == "supplier.txt")
+        .expect("Current supplier source");
+    assert!(supplier.snippet.contains("thirty days"));
+    assert!(supplier.source.is_some());
+
+    // Mutation before indexing must be caught against the persisted exact
+    // parser-input hash, before spending document inference on stale passages.
+    let metadata = fs::metadata(&changing)?;
+    let replacement = "The supplier contract ends after eighty days written notice.";
+    assert_eq!(replacement.len() as u64, metadata.len());
+    fs::write(&changing, replacement)?;
+    fs::File::options()
+        .write(true)
+        .open(&changing)?
+        .set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?;
+    assert_eq!(fs::metadata(&changing)?.modified()?, metadata.modified()?);
+    let error = service.index_embeddings().unwrap_err();
+    assert!(format!("{error:#}").contains("reindex"), "{error:#}");
+    assert_eq!(
+        control.lock().unwrap().calls,
+        4,
+        "Stale extraction must fail before document inference"
+    );
+    assert_eq!(active()?, latest);
+    lexical.index_folder(&root)?;
+    let report = service.index_embeddings()?;
+    assert!(report.resumed);
+    assert_eq!(report.reused_sources, 1);
+    assert_eq!(report.encoded_sources, 1);
+    assert_eq!(control.lock().unwrap().calls, 5);
+    let results = service.search_semantic("supplier contract notice", 10, &options)?;
+    let supplier = results
+        .iter()
+        .find(|r| r.filename == "supplier.txt")
+        .expect("Re-extracted supplier source");
+    assert!(supplier.snippet.contains("eighty days"));
+    assert!(supplier.source.is_some());
+    Ok(())
+}