Verify extraction provenance and prune semantic staging before inference

crates/searchi-core/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 serde.workspace = true
+sha2 = "0.10"
 rusqlite = { version = "0.32", features = ["bundled"] }
 walkdir = "2"
 lopdf = { version = "0.45", default-features = false }

crates/searchi-core/src/extraction.rs

@@ -2,6 +2,7 @@
 use crate::SourceLocation;
 use anyhow::{bail, Context, Result};
 use quick_xml::{events::Event, name::ResolveResult, NsReader, XmlVersion};
+use sha2::{Digest, Sha256};
 use std::{
     fs::File,
     io::{Cursor, Read},
@@ -19,32 +20,25 @@ pub(crate) struct Passage {
     pub source: SourceLocation,
 }
 
-pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
+pub(crate) struct SourceInput {
+    bytes: Vec<u8>,
+    pub hash: String,
+}
+
+/// Read once: the fingerprint and parser consume precisely the same bytes.
+pub(crate) fn read_source(path: &Path) -> Result<Option<SourceInput>> {
     let extension = path
         .extension()
         .and_then(|value| value.to_str())
         .unwrap_or("")
         .to_ascii_lowercase();
-    if extension == "pdf" || extension == "docx" {
-        let bytes = read_bounded(path, MAX_DOCUMENT)?;
-        return if extension == "pdf" {
-            pdf(&bytes).map(Some)
-        } else {
-            docx(&bytes).map(Some)
-        };
-    }
-    let code = crate::is_code_path(path);
-    if !code
+    let document = matches!(extension.as_str(), "pdf" | "docx");
+    if !document
+        && !crate::is_code_path(path)
         && !matches!(
             extension.as_str(),
             "txt"
                 | "md"
-                | "rs"
-                | "py"
-                | "js"
-                | "ts"
-                | "tsx"
-                | "jsx"
                 | "json"
                 | "toml"
                 | "yaml"
@@ -59,16 +53,31 @@ pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
     {
         return Ok(None);
     }
-    let file = File::open(path)?;
-    if file.metadata()?.len() > MAX_TEXT as u64 {
+    if !document && File::open(path)?.metadata()?.len() > MAX_TEXT as u64 {
         return Ok(None);
     }
-    let mut bytes = Vec::new();
-    file.take(MAX_TEXT as u64 + 1).read_to_end(&mut bytes)?;
-    if bytes.len() > MAX_TEXT || bytes.contains(&0) {
+    let bytes = read_bounded(path, if document { MAX_DOCUMENT } else { MAX_TEXT })?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    Ok(Some(SourceInput { bytes, hash }))
+}
+
+pub(crate) fn extract(path: &Path, input: SourceInput) -> Result<Option<Vec<Passage>>> {
+    let extension = path
+        .extension()
+        .and_then(|value| value.to_str())
+        .unwrap_or("")
+        .to_ascii_lowercase();
+    if extension == "pdf" {
+        return pdf(&input.bytes).map(Some);
+    }
+    if extension == "docx" {
+        return docx(&input.bytes).map(Some);
+    }
+    let code = crate::is_code_path(path);
+    if input.bytes.contains(&0) {
         return Ok(None);
     }
-    let Ok(text) = String::from_utf8(bytes) else {
+    let Ok(text) = String::from_utf8(input.bytes) else {
         return Ok(None);
     };
     let lines = text.split_inclusive('\n').collect::<Vec<_>>();
@@ -288,3 +297,24 @@ fn docx(bytes: &[u8]) -> Result<Vec<Passage>> {
     }
     Ok(passages)
 }
+
+#[cfg(test)]
+mod provenance_tests {
+    use super::*;
+
+    #[test]
+    fn parsing_and_fingerprint_share_one_snapshot_even_when_path_changes() -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let path = directory.path().join("notes.txt");
+        std::fs::write(&path, "original parser input")?;
+        let input = read_source(&path)?.unwrap();
+        assert_eq!(
+            input.hash,
+            format!("{:x}", Sha256::digest(b"original parser input"))
+        );
+        std::fs::write(&path, "replacement on disk")?;
+        let passages = extract(&path, input)?.unwrap();
+        assert_eq!(passages[0].text, "original parser input");
+        Ok(())
+    }
+}

crates/searchi-core/src/lib.rs

@@ -21,7 +21,7 @@ use std::{
 };
 use walkdir::WalkDir;
 
-const SCHEMA_VERSION: i64 = 11;
+const SCHEMA_VERSION: i64 = 12;
 const MAX_RESULTS: usize = 1000;
 
 #[derive(Debug, Default, Serialize)]
@@ -255,7 +255,14 @@ impl SearchIndex {
                     [],
                 )?;
             }
-            initialization.execute_batch("PRAGMA user_version=11;")?;
+            let has_extraction_hash: bool = initialization.query_row(
+                "SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='extracted_sha256')",
+                [], |row| row.get(0),
+            )?;
+            if !has_extraction_hash {
+                initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?;
+            }
+            initialization.execute_batch("PRAGMA user_version=12;")?;
             initialization.commit()?;
         }
         Ok(Self {
@@ -265,6 +272,21 @@ impl SearchIndex {
         })
     }
 
+    /// SHA-256 of the exact raw bytes used for the indexed extraction. Legacy
+    /// records return None until reindexed; callers must not infer provenance
+    /// from size/mtime or hash a later file read on their behalf.
+    pub fn extracted_source_hash(&self, path: &str) -> Result<Option<String>> {
+        Ok(self
+            .connection
+            .query_row(
+                "SELECT extracted_sha256 FROM files WHERE path=?1",
+                [path],
+                |row| row.get::<_, Option<String>>(0),
+            )
+            .optional()?
+            .flatten())
+    }
+
     /// Persist genuine extracted media tags, guarded against filesystem changes.
     pub fn set_media_metadata(
         &mut self,
@@ -442,18 +464,46 @@ impl SearchIndex {
                 .and_then(|value| value.to_str())
                 .unwrap_or("")
                 .to_ascii_lowercase();
-            let previous: Option<(i64, i64, String)> = transaction
+            let previous: Option<(i64, i64, String, Option<String>)> = transaction
                 .query_row(
-                    "SELECT size,mtime,root FROM files WHERE path=?1",
+                    "SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
                     [&path],
-                    |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+                    |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
                 )
                 .optional()?;
-            if previous == Some((size, mtime, root_key.clone())) {
+            // Supported inputs are bounded and fingerprinted even when file
+            // metadata is unchanged. This detects same-stat overwrites while
+            // unsupported media/giant files retain the metadata-only fast path.
+            let input = extraction::read_source(entry.path());
+            if matches!(&input, Ok(Some(_))) {
+                let after = fs::metadata(entry.path())?;
+                if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
+                    bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
+                }
+            }
+            let source_hash = input
+                .as_ref()
+                .ok()
+                .and_then(|input| input.as_ref())
+                .map(|input| input.hash.clone());
+            if previous
+                .as_ref()
+                .is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
+                    *old_size == size
+                        && *old_mtime == mtime
+                        && old_root == &root_key
+                        && input.is_ok()
+                        && old_hash == &source_hash
+                })
+            {
                 report.unchanged += 1;
                 continue;
             }
-            let (extracted, extraction_error) = match extraction::extract(entry.path()) {
+            let extracted = input.and_then(|input| match input {
+                Some(input) => extraction::extract(entry.path(), input),
+                None => Ok(None),
+            });
+            let (extracted, extraction_error) = match extracted {
                 Ok(extracted) => (extracted, None),
                 Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
                     report.failures += 1;
@@ -475,11 +525,11 @@ impl SearchIndex {
                 .collect::<Vec<_>>()
                 .join("\n");
             transaction.execute(
-                "INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)
+                "INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
                 ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename,
                 content=excluded.content, size=excluded.size, mtime=excluded.mtime,
-                extension=excluded.extension, extraction_error=excluded.extraction_error",
-                params![path, root_key, filename, content, size, mtime, extension, extraction_error],
+                extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256",
+                params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash],
             )?;
             transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
             for passage in extracted {
@@ -962,8 +1012,8 @@ impl SearchIndex {
     }
 
     /// Read an indexed document source for agent citations or explicit answers.
-    /// No file content is read: the canonical source must still match the indexed
-    /// size and nanosecond modification time. Callers must disclose this bounded
+    /// The canonical source must match indexed metadata and the fingerprint of
+    /// the exact bounded parser input. Callers must disclose this bounded
     /// source excerpt before sending it to an optional external model.
     pub fn source_passage(
         &self,
@@ -993,6 +1043,16 @@ impl SearchIndex {
         let mtime = metadata.modified()?.duration_since(UNIX_EPOCH)?;
         let size = i64::try_from(metadata.len())?;
         let modified = i64::try_from(mtime.as_nanos())?;
+        let Some(expected_hash) = self.extracted_source_hash(path)? else {
+            return Ok(None);
+        };
+        let current_hash = match extraction::read_source(&canonical) {
+            Ok(Some(input)) => input.hash,
+            _ => return Ok(None),
+        };
+        if current_hash != expected_hash {
+            return Ok(None);
+        }
         self.connection
             .query_row(
                 "SELECT p.id,f.path,f.filename,substr(p.text,1,?6),p.kind,p.number

crates/searchi-core/tests/raw_source_hash.rs

@@ -0,0 +1,92 @@
+//! Exact extraction provenance and same-metadata mutation regressions.
+use anyhow::Result;
+use searchi_core::SearchIndex;
+use sha2::{Digest, Sha256};
+use std::{fs, fs::FileTimes};
+
+#[test]
+fn same_stat_overwrite_invalidates_agent_excerpt_and_rescan_reextracts() -> Result<()> {
+    let directory = tempfile::tempdir()?;
+    let path = directory.path().join("source.rs");
+    let original = "fn old_symbol() { /* first */ }\n";
+    let changed = "fn new_symbol() { /* other */ }\n";
+    assert_eq!(original.len(), changed.len());
+    fs::write(&path, original)?;
+    let mut index = SearchIndex::open(":memory:")?;
+    index.index_folder(directory.path())?;
+    let passage = index.passage_batch(0, 1)?.remove(0);
+    assert_eq!(
+        index.extracted_source_hash(&passage.path)?,
+        Some(format!("{:x}", Sha256::digest(original.as_bytes())))
+    );
+    let modified = fs::metadata(&path)?.modified()?;
+    fs::write(&path, changed)?;
+    fs::File::options()
+        .write(true)
+        .open(&path)?
+        .set_times(FileTimes::new().set_modified(modified))?;
+    assert!(index
+        .source_passage(&passage.path, &passage.source, 8192)?
+        .is_none());
+    let report = index.index_folder(directory.path())?;
+    assert_eq!(report.indexed, 1);
+    assert_eq!(report.unchanged, 0);
+    let fresh = index.passage_batch(0, 1)?.remove(0);
+    assert_eq!(fresh.text, changed);
+    assert_eq!(
+        index.extracted_source_hash(&fresh.path)?,
+        Some(format!("{:x}", Sha256::digest(changed.as_bytes())))
+    );
+    assert_eq!(
+        index
+            .source_passage(&fresh.path, &fresh.source, 8192)?
+            .unwrap()
+            .text,
+        changed
+    );
+    assert_eq!(index.index_folder(directory.path())?.unchanged, 1);
+    Ok(())
+}
+
+#[test]
+fn schema_eleven_keeps_lexical_content_but_requires_extraction_provenance() -> Result<()> {
+    let directory = tempfile::tempdir()?;
+    let root = directory.path().join("docs");
+    fs::create_dir(&root)?;
+    fs::write(root.join("notes.txt"), "preserved migration evidence")?;
+    let db = directory.path().join("index.db");
+    let passage;
+    {
+        let mut index = SearchIndex::open(&db)?;
+        index.index_folder(&root)?;
+        passage = index.passage_batch(0, 1)?.remove(0);
+    }
+    let connection = rusqlite::Connection::open(&db)?;
+    connection
+        .execute_batch("ALTER TABLE files DROP COLUMN extracted_sha256; PRAGMA user_version=11;")?;
+    drop(connection);
+    let mut index = SearchIndex::open(&db)?;
+    assert_eq!(index.search("migration", 10)?.len(), 1);
+    assert!(index.extracted_source_hash(&passage.path)?.is_none());
+    assert!(index
+        .source_passage(&passage.path, &passage.source, 8192)?
+        .is_none());
+    assert_eq!(index.index_folder(&root)?.indexed, 1);
+    assert!(index.extracted_source_hash(&passage.path)?.is_some());
+    assert_eq!(index.index_folder(&root)?.unchanged, 1);
+    Ok(())
+}
+
+#[test]
+fn oversized_text_and_unsupported_files_remain_filename_only() -> Result<()> {
+    let directory = tempfile::tempdir()?;
+    let large = directory.path().join("large.txt");
+    fs::File::create(&large)?.set_len(2 * 1024 * 1024 + 1)?;
+    fs::File::create(directory.path().join("movie.mp4"))?.set_len(100 * 1024 * 1024)?;
+    let mut index = SearchIndex::open(":memory:")?;
+    assert_eq!(index.index_folder(directory.path())?.indexed, 2);
+    assert!(index.passage_batch(0, 64)?.is_empty());
+    assert_eq!(index.index_folder(directory.path())?.unchanged, 2);
+    assert_eq!(index.search("large.txt", 10)?.len(), 1);
+    Ok(())
+}

crates/searchi-service/src/lib.rs

@@ -301,7 +301,7 @@ impl HybridSearch {
             "{}:text-only",
             generation_fingerprint(self.embedder.fingerprint())
         );
-        self.rebuild_generation(fingerprint, |service, generation, work| {
+        self.rebuild_generation(fingerprint, None, |service, generation, work| {
             service.build(generation, work)
         })
     }

crates/searchi-service/src/media.rs

@@ -42,7 +42,7 @@ impl HybridSearch {
                 limits.max_file_bytes
             ));
         }
-        self.rebuild_generation(fingerprint, |service, generation, work| {
+        self.rebuild_generation(fingerprint, Some(limits), |service, generation, work| {
             service.build(generation, work)?;
             let mut after = 0;
             let mut media_count = 0usize;

crates/searchi-service/src/pipeline.rs

@@ -53,6 +53,7 @@ impl HybridSearch {
     pub(super) fn rebuild_generation(
         &mut self,
         fingerprint: String,
+        media_limits: Option<&searchi_media::MediaLimits>,
         build: impl FnOnce(&mut Self, i64, &mut BuildWork) -> Result<()>,
     ) -> Result<EmbeddingReport> {
         let lock = self.store.build_lock()?;
@@ -83,28 +84,110 @@ impl HybridSearch {
             media: Vec::new(),
             documents: HashMap::new(),
         };
-        let result = build(self, generation, &mut work).and_then(|()| {
-            for identity in work.media.iter().chain(work.documents.values()) {
-                // Full bytes, not just stat metadata: in-place changes can
-                // preserve size and timestamps, including on mounted volumes.
-                let current = searchi_media::MediaIdentity::read(&identity.path, identity.bytes)?;
-                ensure!(
-                    current.sha256 == identity.sha256
-                        && current.modified_ns == identity.modified_ns,
-                    "Source changed during semantic indexing; retry required"
-                );
-            }
-            self.store
-                .reconcile_tasks(generation, &work.keys, revision)?;
-            self.store.activate_checked(generation, revision)?;
-            Ok(work.report)
-        });
+        let result = self
+            .discover_tasks(media_limits)
+            .and_then(|tasks| {
+                self.store
+                    .prune_before_inference(generation, &tasks, revision)?;
+                build(self, generation, &mut work)
+            })
+            .and_then(|()| {
+                self.store
+                    .reconcile_tasks(generation, &work.keys, revision)?;
+                self.store
+                    .activate_checked_with_validation(generation, revision, || {
+                        for identity in work.media.iter().chain(work.documents.values()) {
+                            // Full bytes, not just stat metadata: in-place changes can
+                            // preserve size and timestamps, including on mounted volumes.
+                            let current =
+                                searchi_media::MediaIdentity::read(&identity.path, identity.bytes)?;
+                            ensure!(
+                                current.sha256 == identity.sha256
+                                    && current.modified_ns == identity.modified_ns,
+                                "Source changed during semantic indexing; retry required"
+                            );
+                        }
+                        Ok(())
+                    })?;
+                Ok(work.report)
+            });
         if result.is_err() {
             self.generation_fingerprint = previous_fingerprint;
         }
         result
     }
 
+    /// Discover source identities in keyset batches before spending the encoder
+    /// budget. This extra byte-verification pass enables safe staging pruning;
+    /// it is indexing I/O, not a foreground latency optimization.
+    fn discover_tasks(
+        &self,
+        media_limits: Option<&searchi_media::MediaLimits>,
+    ) -> Result<Vec<SourceTask>> {
+        let mut tasks = Vec::new();
+        let mut after = 0;
+        loop {
+            let passages = self.lexical.passage_batch(after, 64)?;
+            if passages.is_empty() {
+                break;
+            }
+            for passage in passages {
+                after = passage.id;
+                if passage.text.trim().is_empty() {
+                    continue;
+                }
+                ensure!(
+                    tasks.len() < MAX_SCAN_CHUNKS,
+                    "Semantic source task budget exceeded"
+                );
+                tasks.push(text_task(&passage, &self.generation_fingerprint)?);
+            }
+        }
+        if let Some(limits) = media_limits {
+            let mut after = 0;
+            let mut count = 0;
+            loop {
+                let files = self.store.media_batch(after)?;
+                if files.is_empty() {
+                    break;
+                }
+                for (id, path, _, _) in files {
+                    after = id;
+                    let Some(kind) = searchi_media::classify(Path::new(&path)) else {
+                        continue;
+                    };
+                    count += 1;
+                    ensure!(
+                        count <= 10000,
+                        "Media generation exceeds 10000-file resource budget"
+                    );
+                    ensure!(
+                        tasks.len() < MAX_SCAN_CHUNKS,
+                        "Semantic source task budget exceeded"
+                    );
+                    let identity = searchi_media::MediaIdentity::read(
+                        Path::new(&path),
+                        if kind == searchi_media::MediaKind::Image {
+                            64 * 1024 * 1024
+                        } else {
+                            limits.max_file_bytes
+                        },
+                    );
+                    tasks.push(SourceTask {
+                        key: task_key(&["media", &path]),
+                        path,
+                        source_type: "media".into(),
+                        source_hash: identity.map(|identity| identity.sha256).unwrap_or_default(),
+                        title_hash: String::new(),
+                        policy: self.generation_fingerprint.clone(),
+                        passage_id: -id,
+                    });
+                }
+            }
+        }
+        Ok(tasks)
+    }
+
     pub(super) fn build(&mut self, generation: i64, work: &mut BuildWork) -> Result<()> {
         let mut after_id = 0;
         loop {
@@ -126,6 +209,9 @@ impl HybridSearch {
                         Path::new(&passage.path),
                         32 * 1024 * 1024,
                     )?;
+                    let extracted = self.lexical.extracted_source_hash(&passage.path)?;
+                    ensure!(extracted.as_deref() == Some(identity.sha256.as_str()),
+                        "Document bytes differ from extracted passages; reindex the folder before semantic indexing");
                     work.documents.insert(passage.path.clone(), identity);
                 }
                 let task = text_task(&passage, &self.generation_fingerprint)?;

crates/searchi-service/src/store.rs

@@ -305,10 +305,15 @@ impl Store {
 
     #[cfg(test)]
     pub(crate) fn activate(&mut self, generation: i64) -> Result<()> {
-        self.activate_revision(generation, None)
+        self.activate_revision(generation, None, || Ok(()))
     }
 
-    fn activate_revision(&mut self, generation: i64, revision: Option<i64>) -> Result<()> {
+    fn activate_revision(
+        &mut self,
+        generation: i64,
+        revision: Option<i64>,
+        validate_sources: impl FnOnce() -> Result<()>,
+    ) -> Result<()> {
         // A failed graph build or serialization leaves the old generation active.
         let ann = self.build_ann(generation)?;
         let length = ann.serialized_length();
@@ -319,6 +324,10 @@ impl Store {
         let mut serialized = vec![0_u8; length];
         ann.save_to_buffer(&mut serialized)?;
         let checksum = format!("{:x}", Sha256::digest(&serialized));
+        // Hashing/probing sources can be slow: run after graph preparation but
+        // before acquiring the write transaction. The revision check below
+        // still closes races with indexed changes after this validation.
+        validate_sources()?;
         let transaction = self
             .connection
             .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;

crates/searchi-service/src/store/incremental.rs

@@ -181,6 +181,131 @@ impl Store {
         Ok(())
     }
 
+    /// Reconcile completed staging groups before any encoder call. A bounded
+    /// SQL snapshot preserves renamed donors (including swaps and duplicates)
+    /// while rebuilding only current compatible memberships. Active vectors
+    /// are never touched. Full range/vector validation still happens on reuse.
+    pub(crate) fn prune_before_inference(
+        &mut self,
+        generation: i64,
+        tasks: &[SourceTask],
+        revision: i64,
+    ) -> Result<()> {
+        ensure!(
+            tasks.len() <= MAX_SCAN_CHUNKS,
+            "Generation exceeds source task budget"
+        );
+        for task in tasks {
+            validate_task(task)?;
+        }
+        // The bounded snapshot can still contain 100K x 768d vectors. Keep it
+        // SQLite-backed on disk rather than turning that into a Rust/heap copy.
+        self.connection.pragma_update(None, "temp_store", "FILE")?;
+        let tx = self
+            .connection
+            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        let current: i64 = tx.query_row(
+            "SELECT corpus_revision FROM semantic_meta WHERE singleton=1",
+            [],
+            |r| r.get(0),
+        )?;
+        ensure!(
+            current == revision,
+            "Indexed corpus changed during semantic discovery; retry required"
+        );
+        let active: bool = tx.query_row(
+            "SELECT active_generation=?1 FROM semantic_meta WHERE singleton=1",
+            [generation],
+            |r| r.get::<_, Option<bool>>(0).map(|v| v.unwrap_or(false)),
+        )?;
+        ensure!(!active, "Cannot prune the active semantic generation");
+        let existing: usize = tx.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [generation],
+            |r| r.get(0),
+        )?;
+        ensure!(
+            existing <= MAX_SCAN_CHUNKS,
+            "Staging generation exceeds semantic chunk budget"
+        );
+        tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_discovery_tasks(task_key TEXT PRIMARY KEY,path TEXT NOT NULL,source_type TEXT NOT NULL,source_hash TEXT NOT NULL,title_hash TEXT NOT NULL,policy TEXT NOT NULL,passage_id INTEGER NOT NULL);
+            DELETE FROM semantic_discovery_tasks;
+            CREATE TEMP TABLE IF NOT EXISTS semantic_prune_tasks AS SELECT * FROM semantic_tasks WHERE 0;
+            CREATE TEMP TABLE IF NOT EXISTS semantic_prune_chunks AS SELECT * FROM semantic_chunks WHERE 0;
+            DELETE FROM semantic_prune_tasks; DELETE FROM semantic_prune_chunks;")?;
+        for task in tasks {
+            tx.execute(
+                "INSERT INTO semantic_discovery_tasks VALUES(?1,?2,?3,?4,?5,?6,?7)",
+                params![
+                    task.key,
+                    task.path,
+                    task.source_type,
+                    task.source_hash,
+                    task.title_hash,
+                    task.policy,
+                    task.passage_id
+                ],
+            )?;
+        }
+        tx.execute("INSERT INTO semantic_prune_tasks SELECT * FROM semantic_tasks t WHERE generation=?1 AND state='done' AND source_hash!='' AND expected_chunks IS NOT NULL AND expected_chunks=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key)",[generation])?;
+        tx.execute(
+            "INSERT INTO semantic_prune_chunks SELECT * FROM semantic_chunks WHERE generation=?1",
+            [generation],
+        )?;
+        tx.execute_batch("CREATE UNIQUE INDEX IF NOT EXISTS temp.semantic_prune_task_key ON semantic_prune_tasks(task_key);
+            CREATE INDEX IF NOT EXISTS temp.semantic_prune_reuse ON semantic_prune_tasks(source_hash,title_hash,policy,source_type);
+            CREATE INDEX IF NOT EXISTS temp.semantic_prune_membership ON semantic_prune_chunks(task_key);")?;
+        let mut mappings = Vec::new();
+        {
+            let mut statement = tx.prepare("SELECT d.task_key,COALESCE((SELECT old.task_key FROM semantic_prune_tasks old WHERE old.task_key=d.task_key AND old.source_hash=d.source_hash AND old.title_hash=d.title_hash AND old.policy=d.policy AND old.source_type=d.source_type),(SELECT old.task_key FROM semantic_prune_tasks old WHERE old.source_hash=d.source_hash AND old.title_hash=d.title_hash AND old.policy=d.policy AND old.source_type=d.source_type ORDER BY old.task_key LIMIT 1)) FROM semantic_discovery_tasks d")?;
+            let rows = statement.query_map([], |r| {
+                Ok((r.get::<_, String>(0)?, r.get::<_, Option<String>>(1)?))
+            })?;
+            for row in rows {
+                let (key, donor) = row?;
+                if let Some(donor) = donor {
+                    mappings.push((key, donor));
+                }
+            }
+        }
+        let mut total = 0usize;
+        for (_, donor) in &mappings {
+            let count: usize = tx.query_row(
+                "SELECT expected_chunks FROM semantic_prune_tasks WHERE task_key=?1",
+                [donor],
+                |r| r.get(0),
+            )?;
+            total = total
+                .checked_add(count)
+                .context("Semantic chunk count overflow")?;
+            ensure!(
+                total <= MAX_SCAN_CHUNKS,
+                "Current completed sources exceed semantic chunk budget"
+            );
+        }
+        tx.execute(
+            "DELETE FROM semantic_chunks WHERE generation=?1",
+            [generation],
+        )?;
+        tx.execute(
+            "DELETE FROM semantic_tasks WHERE generation=?1",
+            [generation],
+        )?;
+        for (key, donor) in mappings {
+            tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts,expected_chunks,message)
+                SELECT ?1,d.task_key,d.path,d.source_type,d.source_hash,d.title_hash,d.policy,d.passage_id,'done',old.attempts,old.expected_chunks,NULL FROM semantic_discovery_tasks d JOIN semantic_prune_tasks old ON old.task_key=?3 WHERE d.task_key=?2",params![generation,key,donor])?;
+            tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
+                SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
+        }
+        tx.execute(
+            "DELETE FROM semantic_media_errors WHERE generation=?1",
+            [generation],
+        )?;
+        tx.execute_batch("DELETE FROM semantic_prune_tasks; DELETE FROM semantic_prune_chunks; DELETE FROM semantic_discovery_tasks;")?;
+        tx.commit()?;
+        Ok(())
+    }
+
     pub(crate) fn reconcile_tasks(
         &mut self,
         generation: i64,
@@ -221,9 +346,21 @@ impl Store {
         Ok(())
     }
 
+    #[cfg(test)]
     pub(crate) fn activate_checked(&mut self, generation: i64, revision: i64) -> Result<()> {
+        self.activate_checked_with_validation(generation, revision, || Ok(()))
+    }
+
+    /// Validate live sources after graph preparation, outside the SQLite write
+    /// transaction. Rejection preserves both the active reader and staging.
+    pub(crate) fn activate_checked_with_validation(
+        &mut self,
+        generation: i64,
+        revision: i64,
+        validate_sources: impl FnOnce() -> Result<()>,
+    ) -> Result<()> {
         validate_completion(&self.connection, generation)?;
-        self.activate_revision(generation, Some(revision))
+        self.activate_revision(generation, Some(revision), validate_sources)
     }
 }
 
@@ -468,6 +605,107 @@ mod tests {
         store.complete_source(g, t, 1)
     }
 
+    #[test]
+    fn pre_inference_pruning_preserves_renamed_donors_at_the_chunk_ceiling() -> Result<()> {
+        // Actual 100K membership boundary; vectors are storage-only fixtures.
+        let dir = tempfile::tempdir()?;
+        let root = dir.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "complete source")?;
+        let db = dir.path().join("index.db");
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let p = lexical.passage_batch(0, 1)?.remove(0);
+        let mut store = Store::open(&db, 2)?;
+        let lock = store.build_lock()?;
+        let active = store.start_or_resume("prune-policy", &lock)?.0;
+        insert_complete(&mut store, active, &p, &task(&p))?;
+        store.activate_checked(active, store.revision()?)?;
+        let staging = store.start_or_resume("prune-policy", &lock)?.0;
+        let original = task(&p);
+        insert_complete(&mut store, staging, &p, &original)?;
+        store.connection.execute("WITH RECURSIVE ord(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM ord WHERE n+1<?2)
+            INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
+            SELECT generation,passage_id,n,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key FROM semantic_chunks CROSS JOIN ord WHERE generation=?1 AND ordinal=0",params![staging,MAX_SCAN_CHUNKS as i64])?;
+        store.complete_source(staging, &original, MAX_SCAN_CHUNKS)?;
+        let moved_dir = root.join("moved");
+        std::fs::create_dir(&moved_dir)?;
+        std::fs::rename(root.join("notes.txt"), moved_dir.join("notes.txt"))?;
+        lexical.index_folder(&root)?;
+        let moved = lexical.passage_batch(0, 1)?.remove(0);
+        let moved_task = task(&moved);
+        let revision = store.revision()?;
+        assert!(store
+            .prune_before_inference(staging, std::slice::from_ref(&moved_task), revision - 1)
+            .is_err());
+        let old_count: usize = store.connection.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [staging],
+            |r| r.get(0),
+        )?;
+        assert_eq!(
+            old_count, MAX_SCAN_CHUNKS,
+            "Revision failure must preserve staging"
+        );
+        store.prune_before_inference(staging, std::slice::from_ref(&moved_task), revision)?;
+        store.stage_task(staging, &moved_task)?;
+        assert_eq!(
+            store.reuse_text_source(staging, &moved_task, &moved)?,
+            Some(MAX_SCAN_CHUNKS)
+        );
+        let (count, old_paths): (usize, usize) = store.connection.query_row(
+            "SELECT count(*),sum(path!=?2) FROM semantic_chunks WHERE generation=?1",
+            params![staging, moved.path],
+            |r| Ok((r.get(0)?, r.get(1)?)),
+        )?;
+        assert_eq!((count, old_paths), (MAX_SCAN_CHUNKS, 0));
+        let mut duplicate = moved_task.clone();
+        duplicate.key.push_str("-another-path");
+        duplicate.path.push_str("-another-path");
+        assert!(
+            store
+                .prune_before_inference(staging, &[moved_task.clone(), duplicate], revision)
+                .is_err(),
+            "True current membership over budget must reject atomically"
+        );
+        let count: usize = store.connection.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [staging],
+            |r| r.get(0),
+        )?;
+        assert_eq!(count, MAX_SCAN_CHUNKS);
+        std::fs::write(
+            moved_dir.join("notes.txt"),
+            "replacement source with new content",
+        )?;
+        lexical.index_folder(&root)?;
+        let replacement = lexical.passage_batch(0, 1)?.remove(0);
+        let replacement_task = task(&replacement);
+        store.prune_before_inference(
+            staging,
+            std::slice::from_ref(&replacement_task),
+            store.revision()?,
+        )?;
+        let count: usize = store.connection.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [staging],
+            |r| r.get(0),
+        )?;
+        assert_eq!(
+            count, 0,
+            "Obsolete ceiling-sized group must be removed before replacement inference"
+        );
+        insert_complete(&mut store, staging, &replacement, &replacement_task)?;
+        assert_eq!(store.active_fingerprint()?.as_deref(), Some("prune-policy"));
+        let active_count: usize = store.connection.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [active],
+            |r| r.get(0),
+        )?;
+        assert_eq!(active_count, 1, "Pruning must never remove active vectors");
+        Ok(())
+    }
+
     #[test]
     fn resume_retains_completed_groups_and_discards_unfinished_vectors() -> Result<()> {
         let dir = tempfile::tempdir()?;
@@ -583,6 +821,59 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn late_source_validation_rejects_publication_without_holding_write_lock() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let root = dir.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "source identity before graph build")?;
+        let db = dir.path().join("index.db");
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let passage = lexical.passage_batch(0, 1)?.remove(0);
+        let source = task(&passage);
+        let mut store = Store::open(&db, 2)?;
+        let lock = store.build_lock()?;
+        let (active, _) = store.start_or_resume("old-policy", &lock)?;
+        insert_complete(&mut store, active, &passage, &source)?;
+        store.activate_checked(active, store.revision()?)?;
+        let (staging, _) = store.start_or_resume("new-policy", &lock)?;
+        insert_complete(&mut store, staging, &passage, &source)?;
+        let revision = store.revision()?;
+        let mut called = false;
+        let error = store
+            .activate_checked_with_validation(staging, revision, || {
+                called = true;
+                // An independent write succeeds only when validation runs outside
+                // the publishing connection's Immediate transaction.
+                let other = rusqlite::Connection::open(&db)?;
+                other.busy_timeout(std::time::Duration::ZERO)?;
+                other.execute_batch("BEGIN IMMEDIATE; COMMIT;")?;
+                std::fs::write(root.join("notes.txt"), "direct filesystem mutation")?;
+                anyhow::bail!("source changed after graph preparation")
+            })
+            .unwrap_err();
+        assert!(called);
+        assert!(error
+            .to_string()
+            .contains("source changed after graph preparation"));
+        assert_eq!(store.active_fingerprint()?.as_deref(), Some("old-policy"));
+        assert_eq!(
+            store.ann_cache.as_ref().map(|(generation, _)| *generation),
+            Some(active)
+        );
+        assert_eq!(store.start_or_resume("new-policy", &lock)?, (staging, true));
+        assert_eq!(
+            store.connection.query_row(
+                "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+                [staging],
+                |row| row.get::<_, usize>(0),
+            )?,
+            1
+        );
+        Ok(())
+    }
+
     #[test]
     fn revision_race_cannot_publish_and_failed_activation_keeps_staging() -> Result<()> {
         let dir = tempfile::tempdir()?;

crates/searchi-service/tests/publication_google_cpu.rs

@@ -0,0 +1,189 @@
+//! Publication faults exercised with the actual Google CPU encoder.
+use anyhow::Result;
+use rusqlite::Connection;
+use searchi_core::SearchIndex;
+use searchi_semantic::{GemmaEmbedder, TextWindow};
+use searchi_service::{EmbeddingProvider, HybridSearch, SearchOptions};
+use std::{
+    fs,
+    path::PathBuf,
+    sync::{Arc, Mutex},
+};
+
+#[derive(Default)]
+struct Control {
+    calls: usize,
+    mutation: Option<(PathBuf, String)>,
+}
+struct GoogleWithMutation {
+    google: GemmaEmbedder,
+    control: Arc<Mutex<Control>>,
+}
+impl EmbeddingProvider for GoogleWithMutation {
+    fn fingerprint(&self) -> &str {
+        EmbeddingProvider::fingerprint(&self.google)
+    }
+    fn dimensions(&self) -> usize {
+        EmbeddingProvider::dimensions(&self.google)
+    }
+    fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>> {
+        self.google.split_windows(text, title)
+    }
+    fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result<Vec<Vec<f32>>> {
+        let vectors = self.google.embed_documents(texts, title)?;
+        let mut control = self.control.lock().unwrap();
+        control.calls += texts.len();
+        if let Some((path, replacement)) = control.mutation.take() {
+            let metadata = fs::metadata(&path)?;
+            assert_eq!(metadata.len(), replacement.len() as u64);
+            fs::write(&path, replacement)?;
+            fs::File::options()
+                .write(true)
+                .open(&path)?
+                .set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?;
+            assert_eq!(fs::metadata(&path)?.modified()?, metadata.modified()?);
+        }
+        Ok(vectors)
+    }
+    fn embed_query(&mut self, text: &str) -> Result<Vec<f32>> {
+        self.google.embed_query(text)
+    }
+    fn embed_code_query(&mut self, text: &str) -> Result<Vec<f32>> {
+        self.google.embed_code_query(text)
+    }
+}
+
+#[test]
+#[ignore = "Requires verified Google models and normal native worker; CPU only"]
+fn genuine_google_same_metadata_mutation_rejects_publication_and_preserves_ready_sources(
+) -> Result<()> {
+    assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
+    let google = GemmaEmbedder::load(
+        std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Verified Google assets required"),
+        std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Normal auto worker required"),
+        1,
+    )?;
+    assert_eq!(google.info.device, "cpu");
+    let temp = tempfile::tempdir()?;
+    let root = temp.path().join("documents");
+    fs::create_dir(&root)?;
+    let stable = root.join("garden.txt");
+    fs::write(
+        &stable,
+        "The garden irrigation waters tomato plants every morning.",
+    )?;
+    let changing = root.join("supplier.txt");
+    fs::write(
+        &changing,
+        "The supplier contract ends after sixty days written notice.",
+    )?;
+    let db = temp.path().join("index.db");
+    let mut lexical = SearchIndex::open(&db)?;
+    lexical.index_folder(&root)?;
+    let control = Arc::new(Mutex::new(Control::default()));
+    let mut service = HybridSearch::with_provider(
+        &db,
+        Box::new(GoogleWithMutation {
+            google,
+            control: control.clone(),
+        }),
+    )?;
+    service.index_embeddings()?;
+    let connection = Connection::open(&db)?;
+    let active = || {
+        connection.query_row(
+            "SELECT active_generation FROM semantic_meta WHERE singleton=1",
+            [],
+            |r| r.get::<_, i64>(0),
+        )
+    };
+    let first = active()?;
+    let stable_vector: Vec<u8> = connection.query_row(
+        "SELECT vector FROM semantic_chunks WHERE generation=?1 AND path LIKE '%garden.txt'",
+        [first],
+        |r| r.get(0),
+    )?;
+    fs::write(
+        &changing,
+        "The supplier contract ends after ninety days written notice.",
+    )?;
+    lexical.index_folder(&root)?;
+    control.lock().unwrap().mutation = Some((
+        changing.clone(),
+        "The supplier contract ends after thirty days written notice.".into(),
+    ));
+    let error = service.index_embeddings().unwrap_err();
+    assert!(
+        format!("{error:#}").contains("Source changed during semantic indexing"),
+        "{error:#}"
+    );
+    assert_eq!(
+        active()?,
+        first,
+        "Changed bytes must never publish a stale generation"
+    );
+    assert_eq!(control.lock().unwrap().calls, 3);
+    let options = SearchOptions::default();
+    let results = service.search_semantic("tomato irrigation", 10, &options)?;
+    assert!(results
+        .iter()
+        .any(|r| r.filename == "garden.txt" && r.source.is_some()));
+
+    // A normal rescan must detect changed parser input even though the
+    // mutation restored size and modification time; no metadata workaround.
+    lexical.index_folder(&root)?;
+    let report = service.index_embeddings()?;
+    assert!(report.resumed);
+    assert_eq!(report.reused_sources, 1);
+    assert_eq!(report.encoded_sources, 1);
+    assert_eq!(control.lock().unwrap().calls, 4);
+    let latest = active()?;
+    assert_ne!(latest, first);
+    let latest_stable: Vec<u8> = connection.query_row(
+        "SELECT vector FROM semantic_chunks WHERE generation=?1 AND path LIKE '%garden.txt'",
+        [latest],
+        |r| r.get(0),
+    )?;
+    assert_eq!(latest_stable, stable_vector);
+    let results = service.search_semantic("supplier contract notice", 10, &options)?;
+    let supplier = results
+        .iter()
+        .find(|r| r.filename == "supplier.txt")
+        .expect("Current supplier source");
+    assert!(supplier.snippet.contains("thirty days"));
+    assert!(supplier.source.is_some());
+
+    // Mutation before indexing must be caught against the persisted exact
+    // parser-input hash, before spending document inference on stale passages.
+    let metadata = fs::metadata(&changing)?;
+    let replacement = "The supplier contract ends after eighty days written notice.";
+    assert_eq!(replacement.len() as u64, metadata.len());
+    fs::write(&changing, replacement)?;
+    fs::File::options()
+        .write(true)
+        .open(&changing)?
+        .set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?;
+    assert_eq!(fs::metadata(&changing)?.modified()?, metadata.modified()?);
+    let error = service.index_embeddings().unwrap_err();
+    assert!(format!("{error:#}").contains("reindex"), "{error:#}");
+    assert_eq!(
+        control.lock().unwrap().calls,
+        4,
+        "Stale extraction must fail before document inference"
+    );
+    assert_eq!(active()?, latest);
+    lexical.index_folder(&root)?;
+    let report = service.index_embeddings()?;
+    assert!(report.resumed);
+    assert_eq!(report.reused_sources, 1);
+    assert_eq!(report.encoded_sources, 1);
+    assert_eq!(control.lock().unwrap().calls, 5);
+    let results = service.search_semantic("supplier contract notice", 10, &options)?;
+    let supplier = results
+        .iter()
+        .find(|r| r.filename == "supplier.txt")
+        .expect("Re-extracted supplier source");
+    assert!(supplier.snippet.contains("eighty days"));
+    assert!(supplier.source.is_some());
+    Ok(())
+}