karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Verify extraction provenance and prune semantic staging before inference

Browse Source
This commit is contained in:
karl committed 2026-10-13 01:14:40 +02:00
1 parent 5c18e7b49c
commit 423a1ea57d
17 files changed
+900 -78
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/searchi-core/src/extraction.rs
+53 -23
View File
@@ -2,6 +2,7 @@
use crate::SourceLocation;
use anyhow::{bail, Context, Result};
use quick_xml::{events::Event, name::ResolveResult, NsReader, XmlVersion};
use sha2::{Digest, Sha256};
use std::{
fs::File,
io::{Cursor, Read},
@@ -19,32 +20,25 @@ pub(crate) struct Passage {
pub source: SourceLocation,
}
pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
pub(crate) struct SourceInput {
bytes: Vec<u8>,
pub hash: String,
}
/// Read once: the fingerprint and parser consume precisely the same bytes.
pub(crate) fn read_source(path: &Path) -> Result<Option<SourceInput>> {
let extension = path
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
if extension == "pdf" || extension == "docx" {
let bytes = read_bounded(path, MAX_DOCUMENT)?;
return if extension == "pdf" {
pdf(&bytes).map(Some)
} else {
docx(&bytes).map(Some)
};
}
let code = crate::is_code_path(path);
if !code
let document = matches!(extension.as_str(), "pdf" | "docx");
if !document
&& !crate::is_code_path(path)
&& !matches!(
extension.as_str(),
"txt"
| "md"
| "rs"
| "py"
| "js"
| "ts"
| "tsx"
| "jsx"
| "json"
| "toml"
| "yaml"
@@ -59,16 +53,31 @@ pub(crate) fn extract(path: &Path) -> Result<Option<Vec<Passage>>> {
{
return Ok(None);
}
let file = File::open(path)?;
if file.metadata()?.len() > MAX_TEXT as u64 {
if !document && File::open(path)?.metadata()?.len() > MAX_TEXT as u64 {
return Ok(None);
}
let mut bytes = Vec::new();
file.take(MAX_TEXT as u64 + 1).read_to_end(&mut bytes)?;
if bytes.len() > MAX_TEXT || bytes.contains(&0) {
let bytes = read_bounded(path, if document { MAX_DOCUMENT } else { MAX_TEXT })?;
let hash = format!("{:x}", Sha256::digest(&bytes));
Ok(Some(SourceInput { bytes, hash }))
}
pub(crate) fn extract(path: &Path, input: SourceInput) -> Result<Option<Vec<Passage>>> {
let extension = path
.extension()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
if extension == "pdf" {
return pdf(&input.bytes).map(Some);
}
if extension == "docx" {
return docx(&input.bytes).map(Some);
}
let code = crate::is_code_path(path);
if input.bytes.contains(&0) {
return Ok(None);
}
let Ok(text) = String::from_utf8(bytes) else {
let Ok(text) = String::from_utf8(input.bytes) else {
return Ok(None);
};
let lines = text.split_inclusive('\n').collect::<Vec<_>>();
@@ -288,3 +297,24 @@ fn docx(bytes: &[u8]) -> Result<Vec<Passage>> {
}
Ok(passages)
}
#[cfg(test)]
mod provenance_tests {
use super::*;
#[test]
fn parsing_and_fingerprint_share_one_snapshot_even_when_path_changes() -> Result<()> {
let directory = tempfile::tempdir()?;
let path = directory.path().join("notes.txt");
std::fs::write(&path, "original parser input")?;
let input = read_source(&path)?.unwrap();
assert_eq!(
input.hash,
format!("{:x}", Sha256::digest(b"original parser input"))
);
std::fs::write(&path, "replacement on disk")?;
let passages = extract(&path, input)?.unwrap();
assert_eq!(passages[0].text, "original parser input");
Ok(())
}
}
crates/searchi-core/src/lib.rs
+72 -12
View File
@@ -21,7 +21,7 @@ use std::{
};
use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 11;
const SCHEMA_VERSION: i64 = 12;
const MAX_RESULTS: usize = 1000;
#[derive(Debug, Default, Serialize)]
@@ -255,7 +255,14 @@ impl SearchIndex {
[],
)?;
}
initialization.execute_batch("PRAGMA user_version=11;")?;
let has_extraction_hash: bool = initialization.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='extracted_sha256')",
[], |row| row.get(0),
)?;
if !has_extraction_hash {
initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?;
}
initialization.execute_batch("PRAGMA user_version=12;")?;
initialization.commit()?;
}
Ok(Self {
@@ -265,6 +272,21 @@ impl SearchIndex {
})
}
/// SHA-256 of the exact raw bytes used for the indexed extraction. Legacy
/// records return None until reindexed; callers must not infer provenance
/// from size/mtime or hash a later file read on their behalf.
pub fn extracted_source_hash(&self, path: &str) -> Result<Option<String>> {
Ok(self
.connection
.query_row(
"SELECT extracted_sha256 FROM files WHERE path=?1",
[path],
|row| row.get::<_, Option<String>>(0),
)
.optional()?
.flatten())
}
/// Persist genuine extracted media tags, guarded against filesystem changes.
pub fn set_media_metadata(
&mut self,
@@ -442,18 +464,46 @@ impl SearchIndex {
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let previous: Option<(i64, i64, String)> = transaction
let previous: Option<(i64, i64, String, Option<String>)> = transaction
.query_row(
"SELECT size,mtime,root FROM files WHERE path=?1",
"SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1",
[&path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
if previous == Some((size, mtime, root_key.clone())) {
// Supported inputs are bounded and fingerprinted even when file
// metadata is unchanged. This detects same-stat overwrites while
// unsupported media/giant files retain the metadata-only fast path.
let input = extraction::read_source(entry.path());
if matches!(&input, Ok(Some(_))) {
let after = fs::metadata(entry.path())?;
if after.len() != metadata.len() || after.modified()? != metadata.modified()? {
bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back");
}
}
let source_hash = input
.as_ref()
.ok()
.and_then(|input| input.as_ref())
.map(|input| input.hash.clone());
if previous
.as_ref()
.is_some_and(|(old_size, old_mtime, old_root, old_hash)| {
*old_size == size
&& *old_mtime == mtime
&& old_root == &root_key
&& input.is_ok()
&& old_hash == &source_hash
})
{
report.unchanged += 1;
continue;
}
let (extracted, extraction_error) = match extraction::extract(entry.path()) {
let extracted = input.and_then(|input| match input {
Some(input) => extraction::extract(entry.path(), input),
None => Ok(None),
});
let (extracted, extraction_error) = match extracted {
Ok(extracted) => (extracted, None),
Err(error) if matches!(extension.as_str(), "pdf" | "docx") => {
report.failures += 1;
@@ -475,11 +525,11 @@ impl SearchIndex {
.collect::<Vec<_>>()
.join("\n");
transaction.execute(
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)
"INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename,
content=excluded.content, size=excluded.size, mtime=excluded.mtime,
extension=excluded.extension, extraction_error=excluded.extraction_error",
params![path, root_key, filename, content, size, mtime, extension, extraction_error],
extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256",
params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash],
)?;
transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?;
for passage in extracted {
@@ -962,8 +1012,8 @@ impl SearchIndex {
}
/// Read an indexed document source for agent citations or explicit answers.
/// No file content is read: the canonical source must still match the indexed
/// size and nanosecond modification time. Callers must disclose this bounded
/// The canonical source must match indexed metadata and the fingerprint of
/// the exact bounded parser input. Callers must disclose this bounded
/// source excerpt before sending it to an optional external model.
pub fn source_passage(
&self,
@@ -993,6 +1043,16 @@ impl SearchIndex {
let mtime = metadata.modified()?.duration_since(UNIX_EPOCH)?;
let size = i64::try_from(metadata.len())?;
let modified = i64::try_from(mtime.as_nanos())?;
let Some(expected_hash) = self.extracted_source_hash(path)? else {
return Ok(None);
};
let current_hash = match extraction::read_source(&canonical) {
Ok(Some(input)) => input.hash,
_ => return Ok(None),
};
if current_hash != expected_hash {
return Ok(None);
}
self.connection
.query_row(
"SELECT p.id,f.path,f.filename,substr(p.text,1,?6),p.kind,p.number