Verify extraction provenance and prune semantic staging before inference

crates/searchi-core/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 serde.workspace = true
+sha2 = "0.10"
 rusqlite = { version = "0.32", features = ["bundled"] }
 walkdir = "2"
 lopdf = { version = "0.45", default-features = false }

crates/searchi-core/src/extraction.rs

@@ -2,6 +2,7 @@
 use crate::SourceLocation;
 use anyhow::{bail, Context, Result};
 use quick_xml::{events::Event, name::ResolveResult, NsReader, XmlVersion};
+use sha2::{Digest, Sha256};
 use std::{
     fs::File,
     io::{Cursor, Read},
@@ -19,32 +20,25 @@ pub(crate) struct Passage {
     pub source: SourceLocation,
 }
 
-pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
+pub(crate) struct SourceInput {
+    bytes: Vec<u8>,
+    pub hash: String,
+}
+
+/// Read once: the fingerprint and parser consume precisely the same bytes.
+pub(crate) fn read_source(path: &Path) -> Result<Option<SourceInput>> {
     let extension = path
         .extension()
         .and_then(|value| value.to_str())
         .unwrap_or("")
         .to_ascii_lowercase();
-    if extension == "pdf" || extension == "docx" {
-        let bytes = read_bounded(path, MAX_DOCUMENT)?;
-        return if extension == "pdf" {
-            pdf(&bytes).map(Some)
-        } else {
-            docx(&bytes).map(Some)
-        };
-    }
-    let code = crate::is_code_path(path);
-    if !code
+    let document = matches!(extension.as_str(), "pdf" | "docx");
+    if !document
+        && !crate::is_code_path(path)
         && !matches!(
             extension.as_str(),
             "txt"
                 | "md"
-                | "rs"
-                | "py"
-                | "js"
-                | "ts"
-                | "tsx"
-                | "jsx"
                 | "json"
                 | "toml"
                 | "yaml"
@@ -59,16 +53,31 @@ pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
     {
         return Ok(None);
     }
-    let file = File::open(path)?;
-    if file.metadata()?.len() > MAX_TEXT as u64 {
+    if !document && File::open(path)?.metadata()?.len() > MAX_TEXT as u64 {
         return Ok(None);
     }
-    let mut bytes = Vec::new();
-    file.take(MAX_TEXT as u64 + 1).read_to_end(&mut bytes)?;
-    if bytes.len() > MAX_TEXT || bytes.contains(&0) {
+    let bytes = read_bounded(path, if document { MAX_DOCUMENT } else { MAX_TEXT })?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    Ok(Some(SourceInput { bytes, hash }))
+}
+
+pub(crate) fn extract(path: &Path, input: SourceInput) -> Result<Option<Vec<Passage>>> {
+    let extension = path
+        .extension()
+        .and_then(|value| value.to_str())
+        .unwrap_or("")
+        .to_ascii_lowercase();
+    if extension == "pdf" {
+        return pdf(&input.bytes).map(Some);
+    }
+    if extension == "docx" {
+        return docx(&input.bytes).map(Some);
+    }
+    let code = crate::is_code_path(path);
+    if input.bytes.contains(&0) {
         return Ok(None);
     }
-    let Ok(text) = String::from_utf8(bytes) else {
+    let Ok(text) = String::from_utf8(input.bytes) else {
         return Ok(None);
     };
     let lines = text.split_inclusive('\n').collect::<Vec<_>>();
@@ -288,3 +297,24 @@ fn docx(bytes: &[u8]) -> Result<Vec<Passage>> {
     }
     Ok(passages)
 }
+
+#[cfg(test)]
+mod provenance_tests {
+    use super::*;
+
+    #[test]
+    fn parsing_and_fingerprint_share_one_snapshot_even_when_path_changes() -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let path = directory.path().join("notes.txt");
+        std::fs::write(&path, "original parser input")?;
+        let input = read_source(&path)?.unwrap();
+        assert_eq!(
+            input.hash,
+            format!("{:x}", Sha256::digest(b"original parser input"))
+        );
+        std::fs::write(&path, "replacement on disk")?;
+        let passages = extract(&path, input)?.unwrap();
+        assert_eq!(passages[0].text, "original parser input");
+        Ok(())
+    }
+}

crates/searchi-core/src/lib.rs

@@ -21,7 +21,7 @@ use std::{
 };
 use walkdir::WalkDir;
 
-const SCHEMA_VERSION: i64 = 11;
+const SCHEMA_VERSION: i64 = 12;
 const MAX_RESULTS: usize = 1000;
 
 #[derive(Debug, Default, Serialize)]
@@ -255,7 +255,14 @@ impl SearchIndex {
                     [],
                 )?;
             }
-            initialization.execute_batch("PRAGMA user_version=11;")?;
+            let has_extraction_hash: bool = initialization.query_row(
+                "SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='extracted_sha256')",
+                [], |row| row.get(0),
+            )?;
+            if !has_extraction_hash {
+                initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?;
+            }
+            initialization.execute_batch("PRAGMA user_version=12;")?;
             initialization.commit()?;
         }
         Ok(Self {
@@ -265,6 +272,21 @@ impl SearchIndex {
         })
     }
 
+    /// SHA-256 of the exact raw bytes used for the indexed extraction. Legacy
+    /// records return None until reindexed; callers must not infer provenance
+    /// from size/mtime or hash a later file read on their behalf.
+    pub fn extracted_source_hash(&self, path: &str) -> Result<Option<String>> {
+        Ok(self
+            .connection
+            .query_row(
+                "SELECT extracted_sha256 FROM files WHERE path=?1",
+                [path],
+                |row| row.get::<_, Option<String>>(0),
+            )
+            .optional()?
+            .flatten())
+    }
+
     /// Persist genuine extracted media tags, guarded against filesystem changes.
     pub fn set_media_metadata(
         &mut self,
@@ -442,18 +464,46 @@ impl SearchIndex {
                 .and_then(|value| value.to_str())
                 .unwrap_or("")
                 .to_ascii_lowercase();
-            let previous: Option<(i64, i64, String)> = transaction
+            let previous: Option<(i64, i64, String, Option<String>)> = transaction
                 .query_row(
-                    "SELECT size,mtime,root FROM files WHERE path=?1",
+                    "SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
                     [&path],
-                    |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+                    |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
                 )
                 .optional()?;
-            if previous == Some((size, mtime, root_key.clone())) {
+            // Supported inputs are bounded and fingerprinted even when file
+            // metadata is unchanged. This detects same-stat overwrites while
+            // unsupported media/giant files retain the metadata-only fast path.
+            let input = extraction::read_source(entry.path());
+            if matches!(&input, Ok(Some(_))) {
+                let after = fs::metadata(entry.path())?;
+                if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
+                    bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
+                }
+            }
+            let source_hash = input
+                .as_ref()
+                .ok()
+                .and_then(|input| input.as_ref())
+                .map(|input| input.hash.clone());
+            if previous
+                .as_ref()
+                .is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
+                    *old_size == size
+                        && *old_mtime == mtime
+                        && old_root == &root_key
+                        && input.is_ok()
+                        && old_hash == &source_hash
+                })
+            {
                 report.unchanged += 1;
                 continue;
             }
-            let (extracted, extraction_error) = match extraction::extract(entry.path()) {
+            let extracted = input.and_then(|input| match input {
+                Some(input) => extraction::extract(entry.path(), input),
+                None => Ok(None),
+            });
+            let (extracted, extraction_error) = match extracted {
                 Ok(extracted) => (extracted, None),
                 Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
                     report.failures += 1;
@@ -475,11 +525,11 @@ impl SearchIndex {
                 .collect::<Vec<_>>()
                 .join("\n");
             transaction.execute(
-                "INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)
+                "INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
                 ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename,
                 content=excluded.content, size=excluded.size, mtime=excluded.mtime,
-                extension=excluded.extension, extraction_error=excluded.extraction_error",
-                params![path, root_key, filename, content, size, mtime, extension, extraction_error],
+                extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256",
+                params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash],
             )?;
             transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
             for passage in extracted {
@@ -962,8 +1012,8 @@ impl SearchIndex {
     }
 
     /// Read an indexed document source for agent citations or explicit answers.
-    /// No file content is read: the canonical source must still match the indexed
-    /// size and nanosecond modification time. Callers must disclose this bounded
+    /// The canonical source must match indexed metadata and the fingerprint of
+    /// the exact bounded parser input. Callers must disclose this bounded
     /// source excerpt before sending it to an optional external model.
     pub fn source_passage(
         &self,
@@ -993,6 +1043,16 @@ impl SearchIndex {
         let mtime = metadata.modified()?.duration_since(UNIX_EPOCH)?;
         let size = i64::try_from(metadata.len())?;
         let modified = i64::try_from(mtime.as_nanos())?;
+        let Some(expected_hash) = self.extracted_source_hash(path)? else {
+            return Ok(None);
+        };
+        let current_hash = match extraction::read_source(&canonical) {
+            Ok(Some(input)) => input.hash,
+            _ => return Ok(None),
+        };
+        if current_hash != expected_hash {
+            return Ok(None);
+        }
         self.connection
             .query_row(
                 "SELECT p.id,f.path,f.filename,substr(p.text,1,?6),p.kind,p.number

crates/searchi-core/tests/raw_source_hash.rs

@@ -0,0 +1,92 @@
+//! Exact extraction provenance and same-metadata mutation regressions.
+use anyhow::Result;
+use searchi_core::SearchIndex;
+use sha2::{Digest, Sha256};
+use std::{fs, fs::FileTimes};
+
+#[test]
+fn same_stat_overwrite_invalidates_agent_excerpt_and_rescan_reextracts() -> Result<()> {
+    let directory = tempfile::tempdir()?;
+    let path = directory.path().join("source.rs");
+    let original = "fn old_symbol() { /* first */ }\n";
+    let changed = "fn new_symbol() { /* other */ }\n";
+    assert_eq!(original.len(), changed.len());
+    fs::write(&path, original)?;
+    let mut index = SearchIndex::open(":memory:")?;
+    index.index_folder(directory.path())?;
+    let passage = index.passage_batch(0, 1)?.remove(0);
+    assert_eq!(
+        index.extracted_source_hash(&passage.path)?,
+        Some(format!("{:x}", Sha256::digest(original.as_bytes())))
+    );
+    let modified = fs::metadata(&path)?.modified()?;
+    fs::write(&path, changed)?;
+    fs::File::options()
+        .write(true)
+        .open(&path)?
+        .set_times(FileTimes::new().set_modified(modified))?;
+    assert!(index
+        .source_passage(&passage.path, &passage.source, 8192)?
+        .is_none());
+    let report = index.index_folder(directory.path())?;
+    assert_eq!(report.indexed, 1);
+    assert_eq!(report.unchanged, 0);
+    let fresh = index.passage_batch(0, 1)?.remove(0);
+    assert_eq!(fresh.text, changed);
+    assert_eq!(
+        index.extracted_source_hash(&fresh.path)?,
+        Some(format!("{:x}", Sha256::digest(changed.as_bytes())))
+    );
+    assert_eq!(
+        index
+            .source_passage(&fresh.path, &fresh.source, 8192)?
+            .unwrap()
+            .text,
+        changed
+    );
+    assert_eq!(index.index_folder(directory.path())?.unchanged, 1);
+    Ok(())
+}
+
+#[test]
+fn schema_eleven_keeps_lexical_content_but_requires_extraction_provenance() -> Result<()> {
+    let directory = tempfile::tempdir()?;
+    let root = directory.path().join("docs");
+    fs::create_dir(&root)?;
+    fs::write(root.join("notes.txt"), "preserved migration evidence")?;
+    let db = directory.path().join("index.db");
+    let passage;
+    {
+        let mut index = SearchIndex::open(&db)?;
+        index.index_folder(&root)?;
+        passage = index.passage_batch(0, 1)?.remove(0);
+    }
+    let connection = rusqlite::Connection::open(&db)?;
+    connection
+        .execute_batch("ALTER TABLE files DROP COLUMN extracted_sha256; PRAGMA user_version=11;")?;
+    drop(connection);
+    let mut index = SearchIndex::open(&db)?;
+    assert_eq!(index.search("migration", 10)?.len(), 1);
+    assert!(index.extracted_source_hash(&passage.path)?.is_none());
+    assert!(index
+        .source_passage(&passage.path, &passage.source, 8192)?
+        .is_none());
+    assert_eq!(index.index_folder(&root)?.indexed, 1);
+    assert!(index.extracted_source_hash(&passage.path)?.is_some());
+    assert_eq!(index.index_folder(&root)?.unchanged, 1);
+    Ok(())
+}
+
+#[test]
+fn oversized_text_and_unsupported_files_remain_filename_only() -> Result<()> {
+    let directory = tempfile::tempdir()?;
+    let large = directory.path().join("large.txt");
+    fs::File::create(&large)?.set_len(2 * 1024 * 1024 + 1)?;
+    fs::File::create(directory.path().join("movie.mp4"))?.set_len(100 * 1024 * 1024)?;
+    let mut index = SearchIndex::open(":memory:")?;
+    assert_eq!(index.index_folder(directory.path())?.indexed, 2);
+    assert!(index.passage_batch(0, 64)?.is_empty());
+    assert_eq!(index.index_folder(directory.path())?.unchanged, 2);
+    assert_eq!(index.search("large.txt", 10)?.len(), 1);
+    Ok(())
+}