Verify extraction provenance and prune semantic staging before inference

crates/searchi-core/src/extraction.rs

@@ -2,6 +2,7 @@
 use crate::SourceLocation;
 use anyhow::{bail, Context, Result};
 use quick_xml::{events::Event, name::ResolveResult, NsReader, XmlVersion};
+use sha2::{Digest, Sha256};
 use std::{
     fs::File,
     io::{Cursor, Read},
@@ -19,32 +20,25 @@ pub(crate) struct Passage {
     pub source: SourceLocation,
 }
 
-pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
+pub(crate) struct SourceInput {
+    bytes: Vec<u8>,
+    pub hash: String,
+}
+
+/// Read once: the fingerprint and parser consume precisely the same bytes.
+pub(crate) fn read_source(path: &Path) -> Result<Option<SourceInput>> {
     let extension = path
         .extension()
         .and_then(|value| value.to_str())
         .unwrap_or("")
         .to_ascii_lowercase();
-    if extension == "pdf" || extension == "docx" {
-        let bytes = read_bounded(path, MAX_DOCUMENT)?;
-        return if extension == "pdf" {
-            pdf(&bytes).map(Some)
-        } else {
-            docx(&bytes).map(Some)
-        };
-    }
-    let code = crate::is_code_path(path);
-    if !code
+    let document = matches!(extension.as_str(), "pdf" | "docx");
+    if !document
+        && !crate::is_code_path(path)
         && !matches!(
             extension.as_str(),
             "txt"
                 | "md"
-                | "rs"
-                | "py"
-                | "js"
-                | "ts"
-                | "tsx"
-                | "jsx"
                 | "json"
                 | "toml"
                 | "yaml"
@@ -59,16 +53,31 @@ pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
     {
         return Ok(None);
     }
-    let file = File::open(path)?;
-    if file.metadata()?.len() > MAX_TEXT as u64 {
+    if !document && File::open(path)?.metadata()?.len() > MAX_TEXT as u64 {
         return Ok(None);
     }
-    let mut bytes = Vec::new();
-    file.take(MAX_TEXT as u64 + 1).read_to_end(&mut bytes)?;
-    if bytes.len() > MAX_TEXT || bytes.contains(&0) {
+    let bytes = read_bounded(path, if document { MAX_DOCUMENT } else { MAX_TEXT })?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    Ok(Some(SourceInput { bytes, hash }))
+}
+
+pub(crate) fn extract(path: &Path, input: SourceInput) -> Result<Option<Vec<Passage>>> {
+    let extension = path
+        .extension()
+        .and_then(|value| value.to_str())
+        .unwrap_or("")
+        .to_ascii_lowercase();
+    if extension == "pdf" {
+        return pdf(&input.bytes).map(Some);
+    }
+    if extension == "docx" {
+        return docx(&input.bytes).map(Some);
+    }
+    let code = crate::is_code_path(path);
+    if input.bytes.contains(&0) {
         return Ok(None);
     }
-    let Ok(text) = String::from_utf8(bytes) else {
+    let Ok(text) = String::from_utf8(input.bytes) else {
         return Ok(None);
     };
     let lines = text.split_inclusive('\n').collect::<Vec<_>>();
@@ -288,3 +297,24 @@ fn docx(bytes: &[u8]) -> Result<Vec<Passage>> {
     }
     Ok(passages)
 }
+
+#[cfg(test)]
+mod provenance_tests {
+    use super::*;
+
+    #[test]
+    fn parsing_and_fingerprint_share_one_snapshot_even_when_path_changes() -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let path = directory.path().join("notes.txt");
+        std::fs::write(&path, "original parser input")?;
+        let input = read_source(&path)?.unwrap();
+        assert_eq!(
+            input.hash,
+            format!("{:x}", Sha256::digest(b"original parser input"))
+        );
+        std::fs::write(&path, "replacement on disk")?;
+        let passages = extract(&path, input)?.unwrap();
+        assert_eq!(passages[0].text, "original parser input");
+        Ok(())
+    }
+}