From 423a1ea57d60c413a0915a4b8a34c7f02fb1d889 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 01:14:40 +0200 Subject: [PATCH] Verify extraction provenance and prune semantic staging before inference --- AGENTS.md | 6 + Cargo.lock | 1 + SKILLS.md | 5 +- apps/desktop/src-tauri/Cargo.lock | 1 + crates/searchi-core/Cargo.toml | 1 + crates/searchi-core/src/extraction.rs | 76 +++-- crates/searchi-core/src/lib.rs | 84 ++++- crates/searchi-core/tests/raw_source_hash.rs | 92 ++++++ crates/searchi-service/src/lib.rs | 2 +- crates/searchi-service/src/media.rs | 2 +- crates/searchi-service/src/pipeline.rs | 118 ++++++- crates/searchi-service/src/store.rs | 13 +- .../searchi-service/src/store/incremental.rs | 293 +++++++++++++++++- .../tests/publication_google_cpu.rs | 189 +++++++++++ docs/INCREMENTAL-SEMANTIC-VALIDATION.md | 51 ++- docs/INCREMENTAL-SEMANTIC.md | 32 +- docs/ROADMAP.md | 12 +- 17 files changed, 900 insertions(+), 78 deletions(-) create mode 100644 crates/searchi-core/tests/raw_source_hash.rs create mode 100644 crates/searchi-service/tests/publication_google_cpu.rs diff --git a/AGENTS.md b/AGENTS.md index f55f105..b10a789 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,12 @@ readiness without completed release gates and recorded evidence. - Desktop setup must work out of the box: ship the native worker, automatically download/verify missing Google assets, reuse cached assets, and expose progress/retry in the UI. - Add meaningful tests for retrieval, persistence, and filesystem lifecycle changes. +- Persist SHA-256 of exact parser input bytes. Same-size/mtime overwrites must + be detected on rescans; semantic encoding and agent excerpts must reject + missing or stale extraction provenance. Preserve legacy lexical search. +- Validate sources after ANN preparation, before the publication write + transaction. Prune obsolete staging before inference without dropping + compatible completed work or modifying the active generation. - Keep docs/ROADMAP.md current. Unexecuted platform checks remain unchecked. - Gitea Actions are disabled for this repository at Karl's explicit request (2026-10-12, repeated failure emails). Do not re-enable without his request. diff --git a/Cargo.lock b/Cargo.lock index 341e628..65247f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2268,6 +2268,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "walkdir", "zip", diff --git a/SKILLS.md b/SKILLS.md index 5ec2c22..adba8a6 100644 --- a/SKILLS.md +++ b/SKILLS.md @@ -77,11 +77,12 @@ Incremental reuse/recovery additionally requires verified CPU FFmpeg helpers in SEARCHI_TEST_FFMPEG and SEARCHI_TEST_FFPROBE. Run the actual Google integration: ```sh -cargo test -p searchi-service --test incremental_google_cpu --test google_cpu --locked -- --ignored --nocapture --test-threads=1 +cargo test -p searchi-service --test publication_google_cpu --test incremental_google_cpu --test google_cpu --locked -- --ignored --nocapture --test-threads=1 ``` Completed-group reuse must preserve actual vector bytes and remap current source -citations. Do not substitute simulated vectors or treat zero encoder calls as a +citations. Verify exact parser-input provenance and reject stale extraction before +encoding; legacy records require a normal rescan. Do not substitute simulated vectors or treat zero encoder calls as a foreground latency benchmark. See docs/INCREMENTAL-SEMANTIC.md for remaining gates. Record actual outputs, hardware, corpus size, background preparation, first-input diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index 7771ae1..8c3fd6f 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -3719,6 +3719,7 @@ dependencies = [ "quick-xml", "rusqlite", "serde", + "sha2 0.10.9", "walkdir", "zip", ] diff --git a/crates/searchi-core/Cargo.toml b/crates/searchi-core/Cargo.toml index 30ca744..2eadb19 100644 --- a/crates/searchi-core/Cargo.toml +++ b/crates/searchi-core/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] anyhow.workspace = true serde.workspace = true +sha2 = "0.10" rusqlite = { version = "0.32", features = ["bundled"] } walkdir = "2" lopdf = { version = "0.45", default-features = false } diff --git a/crates/searchi-core/src/extraction.rs b/crates/searchi-core/src/extraction.rs index 167afb1..6000014 100644 --- a/crates/searchi-core/src/extraction.rs +++ b/crates/searchi-core/src/extraction.rs @@ -2,6 +2,7 @@ use crate::SourceLocation; use anyhow::{bail, Context, Result}; use quick_xml::{events::Event, name::ResolveResult, NsReader, XmlVersion}; +use sha2::{Digest, Sha256}; use std::{ fs::File, io::{Cursor, Read}, @@ -19,32 +20,25 @@ pub(crate) struct Passage { pub source: SourceLocation, } -pub(crate) fn extract(path: &Path) -> Result>> { +pub(crate) struct SourceInput { + bytes: Vec, + pub hash: String, +} + +/// Read once: the fingerprint and parser consume precisely the same bytes. +pub(crate) fn read_source(path: &Path) -> Result> { let extension = path .extension() .and_then(|value| value.to_str()) .unwrap_or("") .to_ascii_lowercase(); - if extension == "pdf" || extension == "docx" { - let bytes = read_bounded(path, MAX_DOCUMENT)?; - return if extension == "pdf" { - pdf(&bytes).map(Some) - } else { - docx(&bytes).map(Some) - }; - } - let code = crate::is_code_path(path); - if !code + let document = matches!(extension.as_str(), "pdf" | "docx"); + if !document + && !crate::is_code_path(path) && !matches!( extension.as_str(), "txt" | "md" - | "rs" - | "py" - | "js" - | "ts" - | "tsx" - | "jsx" | "json" | "toml" | "yaml" @@ -59,16 +53,31 @@ pub(crate) fn extract(path: &Path) -> Result>> { { return Ok(None); } - let file = File::open(path)?; - if file.metadata()?.len() > MAX_TEXT as u64 { + if !document && File::open(path)?.metadata()?.len() > MAX_TEXT as u64 { return Ok(None); } - let mut bytes = Vec::new(); - file.take(MAX_TEXT as u64 + 1).read_to_end(&mut bytes)?; - if bytes.len() > MAX_TEXT || bytes.contains(&0) { + let bytes = read_bounded(path, if document { MAX_DOCUMENT } else { MAX_TEXT })?; + let hash = format!("{:x}", Sha256::digest(&bytes)); + Ok(Some(SourceInput { bytes, hash })) +} + +pub(crate) fn extract(path: &Path, input: SourceInput) -> Result>> { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + if extension == "pdf" { + return pdf(&input.bytes).map(Some); + } + if extension == "docx" { + return docx(&input.bytes).map(Some); + } + let code = crate::is_code_path(path); + if input.bytes.contains(&0) { return Ok(None); } - let Ok(text) = String::from_utf8(bytes) else { + let Ok(text) = String::from_utf8(input.bytes) else { return Ok(None); }; let lines = text.split_inclusive('\n').collect::>(); @@ -288,3 +297,24 @@ fn docx(bytes: &[u8]) -> Result> { } Ok(passages) } + +#[cfg(test)] +mod provenance_tests { + use super::*; + + #[test] + fn parsing_and_fingerprint_share_one_snapshot_even_when_path_changes() -> Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("notes.txt"); + std::fs::write(&path, "original parser input")?; + let input = read_source(&path)?.unwrap(); + assert_eq!( + input.hash, + format!("{:x}", Sha256::digest(b"original parser input")) + ); + std::fs::write(&path, "replacement on disk")?; + let passages = extract(&path, input)?.unwrap(); + assert_eq!(passages[0].text, "original parser input"); + Ok(()) + } +} diff --git a/crates/searchi-core/src/lib.rs b/crates/searchi-core/src/lib.rs index a1b4c09..bd7e8bf 100644 --- a/crates/searchi-core/src/lib.rs +++ b/crates/searchi-core/src/lib.rs @@ -21,7 +21,7 @@ use std::{ }; use walkdir::WalkDir; -const SCHEMA_VERSION: i64 = 11; +const SCHEMA_VERSION: i64 = 12; const MAX_RESULTS: usize = 1000; #[derive(Debug, Default, Serialize)] @@ -255,7 +255,14 @@ impl SearchIndex { [], )?; } - initialization.execute_batch("PRAGMA user_version=11;")?; + let has_extraction_hash: bool = initialization.query_row( + "SELECT EXISTS(SELECT 1 FROM pragma_table_info('files') WHERE name='extracted_sha256')", + [], |row| row.get(0), + )?; + if !has_extraction_hash { + initialization.execute("ALTER TABLE files ADD COLUMN extracted_sha256 TEXT", [])?; + } + initialization.execute_batch("PRAGMA user_version=12;")?; initialization.commit()?; } Ok(Self { @@ -265,6 +272,21 @@ impl SearchIndex { }) } + /// SHA-256 of the exact raw bytes used for the indexed extraction. Legacy + /// records return None until reindexed; callers must not infer provenance + /// from size/mtime or hash a later file read on their behalf. + pub fn extracted_source_hash(&self, path: &str) -> Result> { + Ok(self + .connection + .query_row( + "SELECT extracted_sha256 FROM files WHERE path=?1", + [path], + |row| row.get::<_, Option>(0), + ) + .optional()? + .flatten()) + } + /// Persist genuine extracted media tags, guarded against filesystem changes. pub fn set_media_metadata( &mut self, @@ -442,18 +464,46 @@ impl SearchIndex { .and_then(|value| value.to_str()) .unwrap_or("") .to_ascii_lowercase(); - let previous: Option<(i64, i64, String)> = transaction + let previous: Option<(i64, i64, String, Option)> = transaction .query_row( - "SELECT size,mtime,root FROM files WHERE path=?1", + "SELECT size,mtime,root,extracted_sha256 FROM files WHERE path=?1", [&path], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), ) .optional()?; - if previous == Some((size, mtime, root_key.clone())) { + // Supported inputs are bounded and fingerprinted even when file + // metadata is unchanged. This detects same-stat overwrites while + // unsupported media/giant files retain the metadata-only fast path. + let input = extraction::read_source(entry.path()); + if matches!(&input, Ok(Some(_))) { + let after = fs::metadata(entry.path())?; + if after.len() != metadata.len() || after.modified()? != metadata.modified()? { + bail!("Source changed during extraction read; retry indexing: {path}; reconciliation rolled back"); + } + } + let source_hash = input + .as_ref() + .ok() + .and_then(|input| input.as_ref()) + .map(|input| input.hash.clone()); + if previous + .as_ref() + .is_some_and(|(old_size, old_mtime, old_root, old_hash)| { + *old_size == size + && *old_mtime == mtime + && old_root == &root_key + && input.is_ok() + && old_hash == &source_hash + }) + { report.unchanged += 1; continue; } - let (extracted, extraction_error) = match extraction::extract(entry.path()) { + let extracted = input.and_then(|input| match input { + Some(input) => extraction::extract(entry.path(), input), + None => Ok(None), + }); + let (extracted, extraction_error) = match extracted { Ok(extracted) => (extracted, None), Err(error) if matches!(extension.as_str(), "pdf" | "docx") => { report.failures += 1; @@ -475,11 +525,11 @@ impl SearchIndex { .collect::>() .join("\n"); transaction.execute( - "INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error) VALUES (?1,?2,?3,?4,?5,?6,?7,?8) + "INSERT INTO files(path,root,filename,content,size,mtime,extension,extraction_error,extracted_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9) ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename, content=excluded.content, size=excluded.size, mtime=excluded.mtime, - extension=excluded.extension, extraction_error=excluded.extraction_error", - params![path, root_key, filename, content, size, mtime, extension, extraction_error], + extension=excluded.extension, extraction_error=excluded.extraction_error, extracted_sha256=excluded.extracted_sha256", + params![path, root_key, filename, content, size, mtime, extension, extraction_error, source_hash], )?; transaction.execute("DELETE FROM passages WHERE file_path=?1", [&path])?; for passage in extracted { @@ -962,8 +1012,8 @@ impl SearchIndex { } /// Read an indexed document source for agent citations or explicit answers. - /// No file content is read: the canonical source must still match the indexed - /// size and nanosecond modification time. Callers must disclose this bounded + /// The canonical source must match indexed metadata and the fingerprint of + /// the exact bounded parser input. Callers must disclose this bounded /// source excerpt before sending it to an optional external model. pub fn source_passage( &self, @@ -993,6 +1043,16 @@ impl SearchIndex { let mtime = metadata.modified()?.duration_since(UNIX_EPOCH)?; let size = i64::try_from(metadata.len())?; let modified = i64::try_from(mtime.as_nanos())?; + let Some(expected_hash) = self.extracted_source_hash(path)? else { + return Ok(None); + }; + let current_hash = match extraction::read_source(&canonical) { + Ok(Some(input)) => input.hash, + _ => return Ok(None), + }; + if current_hash != expected_hash { + return Ok(None); + } self.connection .query_row( "SELECT p.id,f.path,f.filename,substr(p.text,1,?6),p.kind,p.number diff --git a/crates/searchi-core/tests/raw_source_hash.rs b/crates/searchi-core/tests/raw_source_hash.rs new file mode 100644 index 0000000..19b6894 --- /dev/null +++ b/crates/searchi-core/tests/raw_source_hash.rs @@ -0,0 +1,92 @@ +//! Exact extraction provenance and same-metadata mutation regressions. +use anyhow::Result; +use searchi_core::SearchIndex; +use sha2::{Digest, Sha256}; +use std::{fs, fs::FileTimes}; + +#[test] +fn same_stat_overwrite_invalidates_agent_excerpt_and_rescan_reextracts() -> Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("source.rs"); + let original = "fn old_symbol() { /* first */ }\n"; + let changed = "fn new_symbol() { /* other */ }\n"; + assert_eq!(original.len(), changed.len()); + fs::write(&path, original)?; + let mut index = SearchIndex::open(":memory:")?; + index.index_folder(directory.path())?; + let passage = index.passage_batch(0, 1)?.remove(0); + assert_eq!( + index.extracted_source_hash(&passage.path)?, + Some(format!("{:x}", Sha256::digest(original.as_bytes()))) + ); + let modified = fs::metadata(&path)?.modified()?; + fs::write(&path, changed)?; + fs::File::options() + .write(true) + .open(&path)? + .set_times(FileTimes::new().set_modified(modified))?; + assert!(index + .source_passage(&passage.path, &passage.source, 8192)? + .is_none()); + let report = index.index_folder(directory.path())?; + assert_eq!(report.indexed, 1); + assert_eq!(report.unchanged, 0); + let fresh = index.passage_batch(0, 1)?.remove(0); + assert_eq!(fresh.text, changed); + assert_eq!( + index.extracted_source_hash(&fresh.path)?, + Some(format!("{:x}", Sha256::digest(changed.as_bytes()))) + ); + assert_eq!( + index + .source_passage(&fresh.path, &fresh.source, 8192)? + .unwrap() + .text, + changed + ); + assert_eq!(index.index_folder(directory.path())?.unchanged, 1); + Ok(()) +} + +#[test] +fn schema_eleven_keeps_lexical_content_but_requires_extraction_provenance() -> Result<()> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("docs"); + fs::create_dir(&root)?; + fs::write(root.join("notes.txt"), "preserved migration evidence")?; + let db = directory.path().join("index.db"); + let passage; + { + let mut index = SearchIndex::open(&db)?; + index.index_folder(&root)?; + passage = index.passage_batch(0, 1)?.remove(0); + } + let connection = rusqlite::Connection::open(&db)?; + connection + .execute_batch("ALTER TABLE files DROP COLUMN extracted_sha256; PRAGMA user_version=11;")?; + drop(connection); + let mut index = SearchIndex::open(&db)?; + assert_eq!(index.search("migration", 10)?.len(), 1); + assert!(index.extracted_source_hash(&passage.path)?.is_none()); + assert!(index + .source_passage(&passage.path, &passage.source, 8192)? + .is_none()); + assert_eq!(index.index_folder(&root)?.indexed, 1); + assert!(index.extracted_source_hash(&passage.path)?.is_some()); + assert_eq!(index.index_folder(&root)?.unchanged, 1); + Ok(()) +} + +#[test] +fn oversized_text_and_unsupported_files_remain_filename_only() -> Result<()> { + let directory = tempfile::tempdir()?; + let large = directory.path().join("large.txt"); + fs::File::create(&large)?.set_len(2 * 1024 * 1024 + 1)?; + fs::File::create(directory.path().join("movie.mp4"))?.set_len(100 * 1024 * 1024)?; + let mut index = SearchIndex::open(":memory:")?; + assert_eq!(index.index_folder(directory.path())?.indexed, 2); + assert!(index.passage_batch(0, 64)?.is_empty()); + assert_eq!(index.index_folder(directory.path())?.unchanged, 2); + assert_eq!(index.search("large.txt", 10)?.len(), 1); + Ok(()) +} diff --git a/crates/searchi-service/src/lib.rs b/crates/searchi-service/src/lib.rs index 04871e8..ab19193 100644 --- a/crates/searchi-service/src/lib.rs +++ b/crates/searchi-service/src/lib.rs @@ -301,7 +301,7 @@ impl HybridSearch { "{}:text-only", generation_fingerprint(self.embedder.fingerprint()) ); - self.rebuild_generation(fingerprint, |service, generation, work| { + self.rebuild_generation(fingerprint, None, |service, generation, work| { service.build(generation, work) }) } diff --git a/crates/searchi-service/src/media.rs b/crates/searchi-service/src/media.rs index 1926f60..778e576 100644 --- a/crates/searchi-service/src/media.rs +++ b/crates/searchi-service/src/media.rs @@ -42,7 +42,7 @@ impl HybridSearch { limits.max_file_bytes )); } - self.rebuild_generation(fingerprint, |service, generation, work| { + self.rebuild_generation(fingerprint, Some(limits), |service, generation, work| { service.build(generation, work)?; let mut after = 0; let mut media_count = 0usize; diff --git a/crates/searchi-service/src/pipeline.rs b/crates/searchi-service/src/pipeline.rs index 9a1b641..bb9f718 100644 --- a/crates/searchi-service/src/pipeline.rs +++ b/crates/searchi-service/src/pipeline.rs @@ -53,6 +53,7 @@ impl HybridSearch { pub(super) fn rebuild_generation( &mut self, fingerprint: String, + media_limits: Option<&searchi_media::MediaLimits>, build: impl FnOnce(&mut Self, i64, &mut BuildWork) -> Result<()>, ) -> Result { let lock = self.store.build_lock()?; @@ -83,28 +84,110 @@ impl HybridSearch { media: Vec::new(), documents: HashMap::new(), }; - let result = build(self, generation, &mut work).and_then(|()| { - for identity in work.media.iter().chain(work.documents.values()) { - // Full bytes, not just stat metadata: in-place changes can - // preserve size and timestamps, including on mounted volumes. - let current = searchi_media::MediaIdentity::read(&identity.path, identity.bytes)?; - ensure!( - current.sha256 == identity.sha256 - && current.modified_ns == identity.modified_ns, - "Source changed during semantic indexing; retry required" - ); - } - self.store - .reconcile_tasks(generation, &work.keys, revision)?; - self.store.activate_checked(generation, revision)?; - Ok(work.report) - }); + let result = self + .discover_tasks(media_limits) + .and_then(|tasks| { + self.store + .prune_before_inference(generation, &tasks, revision)?; + build(self, generation, &mut work) + }) + .and_then(|()| { + self.store + .reconcile_tasks(generation, &work.keys, revision)?; + self.store + .activate_checked_with_validation(generation, revision, || { + for identity in work.media.iter().chain(work.documents.values()) { + // Full bytes, not just stat metadata: in-place changes can + // preserve size and timestamps, including on mounted volumes. + let current = + searchi_media::MediaIdentity::read(&identity.path, identity.bytes)?; + ensure!( + current.sha256 == identity.sha256 + && current.modified_ns == identity.modified_ns, + "Source changed during semantic indexing; retry required" + ); + } + Ok(()) + })?; + Ok(work.report) + }); if result.is_err() { self.generation_fingerprint = previous_fingerprint; } result } + /// Discover source identities in keyset batches before spending the encoder + /// budget. This extra byte-verification pass enables safe staging pruning; + /// it is indexing I/O, not a foreground latency optimization. + fn discover_tasks( + &self, + media_limits: Option<&searchi_media::MediaLimits>, + ) -> Result> { + let mut tasks = Vec::new(); + let mut after = 0; + loop { + let passages = self.lexical.passage_batch(after, 64)?; + if passages.is_empty() { + break; + } + for passage in passages { + after = passage.id; + if passage.text.trim().is_empty() { + continue; + } + ensure!( + tasks.len() < MAX_SCAN_CHUNKS, + "Semantic source task budget exceeded" + ); + tasks.push(text_task(&passage, &self.generation_fingerprint)?); + } + } + if let Some(limits) = media_limits { + let mut after = 0; + let mut count = 0; + loop { + let files = self.store.media_batch(after)?; + if files.is_empty() { + break; + } + for (id, path, _, _) in files { + after = id; + let Some(kind) = searchi_media::classify(Path::new(&path)) else { + continue; + }; + count += 1; + ensure!( + count <= 10000, + "Media generation exceeds 10000-file resource budget" + ); + ensure!( + tasks.len() < MAX_SCAN_CHUNKS, + "Semantic source task budget exceeded" + ); + let identity = searchi_media::MediaIdentity::read( + Path::new(&path), + if kind == searchi_media::MediaKind::Image { + 64 * 1024 * 1024 + } else { + limits.max_file_bytes + }, + ); + tasks.push(SourceTask { + key: task_key(&["media", &path]), + path, + source_type: "media".into(), + source_hash: identity.map(|identity| identity.sha256).unwrap_or_default(), + title_hash: String::new(), + policy: self.generation_fingerprint.clone(), + passage_id: -id, + }); + } + } + } + Ok(tasks) + } + pub(super) fn build(&mut self, generation: i64, work: &mut BuildWork) -> Result<()> { let mut after_id = 0; loop { @@ -126,6 +209,9 @@ impl HybridSearch { Path::new(&passage.path), 32 * 1024 * 1024, )?; + let extracted = self.lexical.extracted_source_hash(&passage.path)?; + ensure!(extracted.as_deref() == Some(identity.sha256.as_str()), + "Document bytes differ from extracted passages; reindex the folder before semantic indexing"); work.documents.insert(passage.path.clone(), identity); } let task = text_task(&passage, &self.generation_fingerprint)?; diff --git a/crates/searchi-service/src/store.rs b/crates/searchi-service/src/store.rs index 397f2fa..dc3c3ff 100644 --- a/crates/searchi-service/src/store.rs +++ b/crates/searchi-service/src/store.rs @@ -305,10 +305,15 @@ impl Store { #[cfg(test)] pub(crate) fn activate(&mut self, generation: i64) -> Result<()> { - self.activate_revision(generation, None) + self.activate_revision(generation, None, || Ok(())) } - fn activate_revision(&mut self, generation: i64, revision: Option) -> Result<()> { + fn activate_revision( + &mut self, + generation: i64, + revision: Option, + validate_sources: impl FnOnce() -> Result<()>, + ) -> Result<()> { // A failed graph build or serialization leaves the old generation active. let ann = self.build_ann(generation)?; let length = ann.serialized_length(); @@ -319,6 +324,10 @@ impl Store { let mut serialized = vec![0_u8; length]; ann.save_to_buffer(&mut serialized)?; let checksum = format!("{:x}", Sha256::digest(&serialized)); + // Hashing/probing sources can be slow: run after graph preparation but + // before acquiring the write transaction. The revision check below + // still closes races with indexed changes after this validation. + validate_sources()?; let transaction = self .connection .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; diff --git a/crates/searchi-service/src/store/incremental.rs b/crates/searchi-service/src/store/incremental.rs index cfe3449..0837685 100644 --- a/crates/searchi-service/src/store/incremental.rs +++ b/crates/searchi-service/src/store/incremental.rs @@ -181,6 +181,131 @@ impl Store { Ok(()) } + /// Reconcile completed staging groups before any encoder call. A bounded + /// SQL snapshot preserves renamed donors (including swaps and duplicates) + /// while rebuilding only current compatible memberships. Active vectors + /// are never touched. Full range/vector validation still happens on reuse. + pub(crate) fn prune_before_inference( + &mut self, + generation: i64, + tasks: &[SourceTask], + revision: i64, + ) -> Result<()> { + ensure!( + tasks.len() <= MAX_SCAN_CHUNKS, + "Generation exceeds source task budget" + ); + for task in tasks { + validate_task(task)?; + } + // The bounded snapshot can still contain 100K x 768d vectors. Keep it + // SQLite-backed on disk rather than turning that into a Rust/heap copy. + self.connection.pragma_update(None, "temp_store", "FILE")?; + let tx = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let current: i64 = tx.query_row( + "SELECT corpus_revision FROM semantic_meta WHERE singleton=1", + [], + |r| r.get(0), + )?; + ensure!( + current == revision, + "Indexed corpus changed during semantic discovery; retry required" + ); + let active: bool = tx.query_row( + "SELECT active_generation=?1 FROM semantic_meta WHERE singleton=1", + [generation], + |r| r.get::<_, Option>(0).map(|v| v.unwrap_or(false)), + )?; + ensure!(!active, "Cannot prune the active semantic generation"); + let existing: usize = tx.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [generation], + |r| r.get(0), + )?; + ensure!( + existing <= MAX_SCAN_CHUNKS, + "Staging generation exceeds semantic chunk budget" + ); + tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_discovery_tasks(task_key TEXT PRIMARY KEY,path TEXT NOT NULL,source_type TEXT NOT NULL,source_hash TEXT NOT NULL,title_hash TEXT NOT NULL,policy TEXT NOT NULL,passage_id INTEGER NOT NULL); + DELETE FROM semantic_discovery_tasks; + CREATE TEMP TABLE IF NOT EXISTS semantic_prune_tasks AS SELECT * FROM semantic_tasks WHERE 0; + CREATE TEMP TABLE IF NOT EXISTS semantic_prune_chunks AS SELECT * FROM semantic_chunks WHERE 0; + DELETE FROM semantic_prune_tasks; DELETE FROM semantic_prune_chunks;")?; + for task in tasks { + tx.execute( + "INSERT INTO semantic_discovery_tasks VALUES(?1,?2,?3,?4,?5,?6,?7)", + params![ + task.key, + task.path, + task.source_type, + task.source_hash, + task.title_hash, + task.policy, + task.passage_id + ], + )?; + } + tx.execute("INSERT INTO semantic_prune_tasks SELECT * FROM semantic_tasks t WHERE generation=?1 AND state='done' AND source_hash!='' AND expected_chunks IS NOT NULL AND expected_chunks=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key)",[generation])?; + tx.execute( + "INSERT INTO semantic_prune_chunks SELECT * FROM semantic_chunks WHERE generation=?1", + [generation], + )?; + tx.execute_batch("CREATE UNIQUE INDEX IF NOT EXISTS temp.semantic_prune_task_key ON semantic_prune_tasks(task_key); + CREATE INDEX IF NOT EXISTS temp.semantic_prune_reuse ON semantic_prune_tasks(source_hash,title_hash,policy,source_type); + CREATE INDEX IF NOT EXISTS temp.semantic_prune_membership ON semantic_prune_chunks(task_key);")?; + let mut mappings = Vec::new(); + { + let mut statement = tx.prepare("SELECT d.task_key,COALESCE((SELECT old.task_key FROM semantic_prune_tasks old WHERE old.task_key=d.task_key AND old.source_hash=d.source_hash AND old.title_hash=d.title_hash AND old.policy=d.policy AND old.source_type=d.source_type),(SELECT old.task_key FROM semantic_prune_tasks old WHERE old.source_hash=d.source_hash AND old.title_hash=d.title_hash AND old.policy=d.policy AND old.source_type=d.source_type ORDER BY old.task_key LIMIT 1)) FROM semantic_discovery_tasks d")?; + let rows = statement.query_map([], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)) + })?; + for row in rows { + let (key, donor) = row?; + if let Some(donor) = donor { + mappings.push((key, donor)); + } + } + } + let mut total = 0usize; + for (_, donor) in &mappings { + let count: usize = tx.query_row( + "SELECT expected_chunks FROM semantic_prune_tasks WHERE task_key=?1", + [donor], + |r| r.get(0), + )?; + total = total + .checked_add(count) + .context("Semantic chunk count overflow")?; + ensure!( + total <= MAX_SCAN_CHUNKS, + "Current completed sources exceed semantic chunk budget" + ); + } + tx.execute( + "DELETE FROM semantic_chunks WHERE generation=?1", + [generation], + )?; + tx.execute( + "DELETE FROM semantic_tasks WHERE generation=?1", + [generation], + )?; + for (key, donor) in mappings { + tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts,expected_chunks,message) + SELECT ?1,d.task_key,d.path,d.source_type,d.source_hash,d.title_hash,d.policy,d.passage_id,'done',old.attempts,old.expected_chunks,NULL FROM semantic_discovery_tasks d JOIN semantic_prune_tasks old ON old.task_key=?3 WHERE d.task_key=?2",params![generation,key,donor])?; + tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key) + SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?; + } + tx.execute( + "DELETE FROM semantic_media_errors WHERE generation=?1", + [generation], + )?; + tx.execute_batch("DELETE FROM semantic_prune_tasks; DELETE FROM semantic_prune_chunks; DELETE FROM semantic_discovery_tasks;")?; + tx.commit()?; + Ok(()) + } + pub(crate) fn reconcile_tasks( &mut self, generation: i64, @@ -221,9 +346,21 @@ impl Store { Ok(()) } + #[cfg(test)] pub(crate) fn activate_checked(&mut self, generation: i64, revision: i64) -> Result<()> { + self.activate_checked_with_validation(generation, revision, || Ok(())) + } + + /// Validate live sources after graph preparation, outside the SQLite write + /// transaction. Rejection preserves both the active reader and staging. + pub(crate) fn activate_checked_with_validation( + &mut self, + generation: i64, + revision: i64, + validate_sources: impl FnOnce() -> Result<()>, + ) -> Result<()> { validate_completion(&self.connection, generation)?; - self.activate_revision(generation, Some(revision)) + self.activate_revision(generation, Some(revision), validate_sources) } } @@ -468,6 +605,107 @@ mod tests { store.complete_source(g, t, 1) } + #[test] + fn pre_inference_pruning_preserves_renamed_donors_at_the_chunk_ceiling() -> Result<()> { + // Actual 100K membership boundary; vectors are storage-only fixtures. + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "complete source")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let p = lexical.passage_batch(0, 1)?.remove(0); + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + let active = store.start_or_resume("prune-policy", &lock)?.0; + insert_complete(&mut store, active, &p, &task(&p))?; + store.activate_checked(active, store.revision()?)?; + let staging = store.start_or_resume("prune-policy", &lock)?.0; + let original = task(&p); + insert_complete(&mut store, staging, &p, &original)?; + store.connection.execute("WITH RECURSIVE ord(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM ord WHERE n+1 Result<()> { let dir = tempfile::tempdir()?; @@ -583,6 +821,59 @@ mod tests { Ok(()) } + #[test] + fn late_source_validation_rejects_publication_without_holding_write_lock() -> Result<()> { + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "source identity before graph build")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let passage = lexical.passage_batch(0, 1)?.remove(0); + let source = task(&passage); + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + let (active, _) = store.start_or_resume("old-policy", &lock)?; + insert_complete(&mut store, active, &passage, &source)?; + store.activate_checked(active, store.revision()?)?; + let (staging, _) = store.start_or_resume("new-policy", &lock)?; + insert_complete(&mut store, staging, &passage, &source)?; + let revision = store.revision()?; + let mut called = false; + let error = store + .activate_checked_with_validation(staging, revision, || { + called = true; + // An independent write succeeds only when validation runs outside + // the publishing connection's Immediate transaction. + let other = rusqlite::Connection::open(&db)?; + other.busy_timeout(std::time::Duration::ZERO)?; + other.execute_batch("BEGIN IMMEDIATE; COMMIT;")?; + std::fs::write(root.join("notes.txt"), "direct filesystem mutation")?; + anyhow::bail!("source changed after graph preparation") + }) + .unwrap_err(); + assert!(called); + assert!(error + .to_string() + .contains("source changed after graph preparation")); + assert_eq!(store.active_fingerprint()?.as_deref(), Some("old-policy")); + assert_eq!( + store.ann_cache.as_ref().map(|(generation, _)| *generation), + Some(active) + ); + assert_eq!(store.start_or_resume("new-policy", &lock)?, (staging, true)); + assert_eq!( + store.connection.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [staging], + |row| row.get::<_, usize>(0), + )?, + 1 + ); + Ok(()) + } + #[test] fn revision_race_cannot_publish_and_failed_activation_keeps_staging() -> Result<()> { let dir = tempfile::tempdir()?; diff --git a/crates/searchi-service/tests/publication_google_cpu.rs b/crates/searchi-service/tests/publication_google_cpu.rs new file mode 100644 index 0000000..7cb6076 --- /dev/null +++ b/crates/searchi-service/tests/publication_google_cpu.rs @@ -0,0 +1,189 @@ +//! Publication faults exercised with the actual Google CPU encoder. +use anyhow::Result; +use rusqlite::Connection; +use searchi_core::SearchIndex; +use searchi_semantic::{GemmaEmbedder, TextWindow}; +use searchi_service::{EmbeddingProvider, HybridSearch, SearchOptions}; +use std::{ + fs, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +#[derive(Default)] +struct Control { + calls: usize, + mutation: Option<(PathBuf, String)>, +} +struct GoogleWithMutation { + google: GemmaEmbedder, + control: Arc>, +} +impl EmbeddingProvider for GoogleWithMutation { + fn fingerprint(&self) -> &str { + EmbeddingProvider::fingerprint(&self.google) + } + fn dimensions(&self) -> usize { + EmbeddingProvider::dimensions(&self.google) + } + fn split_windows(&mut self, text: &str, title: &str) -> Result> { + self.google.split_windows(text, title) + } + fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result>> { + let vectors = self.google.embed_documents(texts, title)?; + let mut control = self.control.lock().unwrap(); + control.calls += texts.len(); + if let Some((path, replacement)) = control.mutation.take() { + let metadata = fs::metadata(&path)?; + assert_eq!(metadata.len(), replacement.len() as u64); + fs::write(&path, replacement)?; + fs::File::options() + .write(true) + .open(&path)? + .set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?; + assert_eq!(fs::metadata(&path)?.modified()?, metadata.modified()?); + } + Ok(vectors) + } + fn embed_query(&mut self, text: &str) -> Result> { + self.google.embed_query(text) + } + fn embed_code_query(&mut self, text: &str) -> Result> { + self.google.embed_code_query(text) + } +} + +#[test] +#[ignore = "Requires verified Google models and normal native worker; CPU only"] +fn genuine_google_same_metadata_mutation_rejects_publication_and_preserves_ready_sources( +) -> Result<()> { + assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1"); + let google = GemmaEmbedder::load( + std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Verified Google assets required"), + std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Normal auto worker required"), + 1, + )?; + assert_eq!(google.info.device, "cpu"); + let temp = tempfile::tempdir()?; + let root = temp.path().join("documents"); + fs::create_dir(&root)?; + let stable = root.join("garden.txt"); + fs::write( + &stable, + "The garden irrigation waters tomato plants every morning.", + )?; + let changing = root.join("supplier.txt"); + fs::write( + &changing, + "The supplier contract ends after sixty days written notice.", + )?; + let db = temp.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let control = Arc::new(Mutex::new(Control::default())); + let mut service = HybridSearch::with_provider( + &db, + Box::new(GoogleWithMutation { + google, + control: control.clone(), + }), + )?; + service.index_embeddings()?; + let connection = Connection::open(&db)?; + let active = || { + connection.query_row( + "SELECT active_generation FROM semantic_meta WHERE singleton=1", + [], + |r| r.get::<_, i64>(0), + ) + }; + let first = active()?; + let stable_vector: Vec = connection.query_row( + "SELECT vector FROM semantic_chunks WHERE generation=?1 AND path LIKE '%garden.txt'", + [first], + |r| r.get(0), + )?; + fs::write( + &changing, + "The supplier contract ends after ninety days written notice.", + )?; + lexical.index_folder(&root)?; + control.lock().unwrap().mutation = Some(( + changing.clone(), + "The supplier contract ends after thirty days written notice.".into(), + )); + let error = service.index_embeddings().unwrap_err(); + assert!( + format!("{error:#}").contains("Source changed during semantic indexing"), + "{error:#}" + ); + assert_eq!( + active()?, + first, + "Changed bytes must never publish a stale generation" + ); + assert_eq!(control.lock().unwrap().calls, 3); + let options = SearchOptions::default(); + let results = service.search_semantic("tomato irrigation", 10, &options)?; + assert!(results + .iter() + .any(|r| r.filename == "garden.txt" && r.source.is_some())); + + // A normal rescan must detect changed parser input even though the + // mutation restored size and modification time; no metadata workaround. + lexical.index_folder(&root)?; + let report = service.index_embeddings()?; + assert!(report.resumed); + assert_eq!(report.reused_sources, 1); + assert_eq!(report.encoded_sources, 1); + assert_eq!(control.lock().unwrap().calls, 4); + let latest = active()?; + assert_ne!(latest, first); + let latest_stable: Vec = connection.query_row( + "SELECT vector FROM semantic_chunks WHERE generation=?1 AND path LIKE '%garden.txt'", + [latest], + |r| r.get(0), + )?; + assert_eq!(latest_stable, stable_vector); + let results = service.search_semantic("supplier contract notice", 10, &options)?; + let supplier = results + .iter() + .find(|r| r.filename == "supplier.txt") + .expect("Current supplier source"); + assert!(supplier.snippet.contains("thirty days")); + assert!(supplier.source.is_some()); + + // Mutation before indexing must be caught against the persisted exact + // parser-input hash, before spending document inference on stale passages. + let metadata = fs::metadata(&changing)?; + let replacement = "The supplier contract ends after eighty days written notice."; + assert_eq!(replacement.len() as u64, metadata.len()); + fs::write(&changing, replacement)?; + fs::File::options() + .write(true) + .open(&changing)? + .set_times(fs::FileTimes::new().set_modified(metadata.modified()?))?; + assert_eq!(fs::metadata(&changing)?.modified()?, metadata.modified()?); + let error = service.index_embeddings().unwrap_err(); + assert!(format!("{error:#}").contains("reindex"), "{error:#}"); + assert_eq!( + control.lock().unwrap().calls, + 4, + "Stale extraction must fail before document inference" + ); + assert_eq!(active()?, latest); + lexical.index_folder(&root)?; + let report = service.index_embeddings()?; + assert!(report.resumed); + assert_eq!(report.reused_sources, 1); + assert_eq!(report.encoded_sources, 1); + assert_eq!(control.lock().unwrap().calls, 5); + let results = service.search_semantic("supplier contract notice", 10, &options)?; + let supplier = results + .iter() + .find(|r| r.filename == "supplier.txt") + .expect("Re-extracted supplier source"); + assert!(supplier.snippet.contains("eighty days")); + assert!(supplier.source.is_some()); + Ok(()) +} diff --git a/docs/INCREMENTAL-SEMANTIC-VALIDATION.md b/docs/INCREMENTAL-SEMANTIC-VALIDATION.md index 20ef61f..b791f86 100644 --- a/docs/INCREMENTAL-SEMANTIC-VALIDATION.md +++ b/docs/INCREMENTAL-SEMANTIC-VALIDATION.md @@ -36,7 +36,7 @@ transactional completion failure and malformed stored text windows. Another fault-only pipeline test covers primary inference error preservation when partial-source cleanup also fails. Its provider returns no simulated vectors. -## Executed checks +## Original reuse commit checks (5c18e7b) - Windows workspace: 98 passed, 15 explicitly ignored; no failures. - Native Ubuntu 24.04 headless workspace: 98 passed, 15 explicitly ignored. @@ -53,11 +53,50 @@ partial-source cleanup also fails. Its provider returns no simulated vectors. isolated verified models/normal auto worker and pinned native FFmpeg helpers. Cold Clippy compilation ran concurrently; these are not latency comparisons. +## Publication and provenance follow-up + +The follow-up moves full source validation after ANN construction/serialization, +before opening the publication write transaction. A unit regression confirms +an independent writer can acquire the database while validation runs; rejection +keeps the previous fingerprint/cache and completed resumable staging. + +Pre-inference pruning uses a bounded, disk-backed transactional SQLite snapshot. +The 100,000-chunk regression checks renamed completed staging without duplicate +allocation, genuine duplicate current groups exceeding the budget, rollback, +obsolete replacement cleanup and preservation of active vectors. This is a +storage fixture, not an ANN latency or neural relevance benchmark. + +Core schema 12 hashes the exact bounded parser input, without reopening the file +for parsing. Four tests cover immutable input snapshots, same-size/mtime mutation, +real schema-11 migration preserving lexical content, stale agent excerpts and +filename-only oversized/unsupported inputs. Supported rescans read and hash +bounded bytes even when metadata is unchanged; matching hashes skip parsing. +Legacy rows need reindexing to establish provenance before semantic rebuilding +or serving agent excerpts. + +## Follow-up executed checks + +- Windows and native Linux workspaces: 104 passed each, 16 explicitly ignored; no failures. +- Windows desktop: 17 passed. +- Strict Windows/Linux workspace and Windows desktop Clippy passed; workspace formatting passed. +- Six genuine Windows Google CPU tests passed: existing lifecycle/policy tests + 32.52 s; mixed incremental tests 182.03 s; new publication mutation test + 11.02 s. Compiler/resource contention was present. These are correctness-suite + times and are not search-latency measurements. +- The same six genuine native Linux Google CPU tests passed: lifecycle/policy + 40.16 s, incremental/media 135.42 s, publication mutation 11.98 s. These + elapsed suites are correctness evidence, not comparative performance claims. +- The mutation test overwrites the file after actual document inference, + restores the exact mtime, rejects publication and preserves the active ID. + An ordinary folder rescan then resumes with only the changed source encoded. + Another overwrite before rebuilding rejects stale extraction before any + additional document encoder calls. Unaffected stored vector bytes and + citations remain identical. + ## Remaining gates -Actual process termination during a multi-batch source, simultaneous real-model -source mutation, wider policy changes, near-limit pruning, minimum-ABI/older-CPU -checks and first-query/foreground latency during indexing remain open. Full file -hash validation currently precedes ANN construction; direct file mutation during -that construction before watcher reconciliation needs a later validation callback. +Actual process termination during a multi-batch source, wider policy changes, +minimum-ABI/older-CPU checks and first-query/foreground latency during indexing +remain open. Live filesystem reads and SQLite publication are not one atomic +operation; the later check narrows that race rather than eliminating it. See [INCREMENTAL-SEMANTIC.md](INCREMENTAL-SEMANTIC.md). diff --git a/docs/INCREMENTAL-SEMANTIC.md b/docs/INCREMENTAL-SEMANTIC.md index d623d7b..4c8ef8e 100644 --- a/docs/INCREMENTAL-SEMANTIC.md +++ b/docs/INCREMENTAL-SEMANTIC.md @@ -69,8 +69,11 @@ preserve both the previous active generation and the same reader's policy. text groups when component encoder-key reuse is implemented. - [ ] Terminate after a committed batch and resume without repeating completed inference. Never reuse or publish an unfinished source group. -- [ ] Mutate or delete a queued/running source and prove stale vectors cannot - activate or become readable. +- [x] Same-size/mtime text mutation during genuine inference rejects publication; + a mutation before rebuilding fails before text encoding and ordinary rescans + reextract it. Unaffected vectors survive resumed retry. +- [ ] Concurrent deletion/media mutation and comprehensive queued/running source + search freshness tests. - [x] Inject activation failure, preserve existing results and resume completed staging with zero document inference; cleanup-failure unit regression retains the primary error and old reader policy. @@ -90,16 +93,23 @@ across media sampling changes remains open. Durable staging resumes at completed source boundaries; a real process-kill test during a multi-batch source remains open. Storage tests cover discarded partial groups and corpus revision races. -Full source hashes are checked before ANN construction; indexed changes during -ANN construction are rejected by the final transaction's revision guard. A direct -filesystem mutation during ANN construction before watcher reconciliation still -needs a later validation callback and concurrent real-model regression. +Full source hashes are checked after ANN construction and serialization, before +opening the publication write transaction. Indexed changes are also rejected by +the transaction's revision guard. This narrows the filesystem race; it does not +make file reads and SQLite publication one atomic filesystem operation. -Obsolete completed groups in resumed staging are removed at final reconciliation. -Near the 100,000-chunk ceiling, replacement work can temporarily hit the budget -before this cleanup and fail safely. Bounded discovery/pruning before inference -is the next improvement. Hashing, media metadata probes and graph rebuilding still -have costs: zero encoder calls does not mean zero work or prove fast first input. +Core schema 12 records the SHA-256 of the exact bounded bytes consumed by each +parser. Semantic builds verify that provenance before reusing or encoding text. +Same-size/mtime changes are detected by normal folder rescanning. Legacy records +retain lexical search but need a rescan before semantic rebuilding or serving +agent excerpts. Excerpts reject missing or mismatched source provenance. + +Bounded discovery now prunes obsolete resumed groups before inference. Compatible +renamed staging groups survive with current memberships through a transactional, +disk-backed SQLite snapshot; active vectors are untouched. Genuine duplicate +current groups still count toward the 100,000-chunk ceiling. Hashing, discovery, +media metadata probes, snapshot copying and graph rebuilding have costs: zero +encoder calls does not mean zero work or prove fast first input. Synthetic colored-image and sine-wave fixtures prove actual vector reuse and citation equality; they do not establish useful media relevance. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 534e1fc..8f8bd4b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -100,7 +100,7 @@ Status: implementation started; no production release yet. Windows VM tests use Warm p95 alone does not satisfy the first-use requirement. - [ ] Test migrations, interrupted work, invalid documents, inaccessible/offline roots and cloud placeholders. - [ ] Automated native build matrix and documented supported cross-compilation paths. -- [x] Native Linux headless workspace tests: 98 passed, 15 explicitly ignored. +- [x] Native Linux headless workspace tests: 104 passed, 16 explicitly ignored. - [x] Native Linux Google text/code/media reference parity and service lifecycle/rebuild recovery: seven genuine CPU integration tests passed. - [x] Unsigned Linux x86-64 headless archive: verify extracted checksums/modes, @@ -147,8 +147,14 @@ Audit and measured priorities: [EMBEDDINGGEMMA2-AUDIT.md](EMBEDDINGGEMMA2-AUDIT. with genuine Google CPU document/code/image/audio/video fixtures. - [x] Resume completed staging sources after failed publication; incomplete groups are purged under the OS build lock before retry. -- [ ] Process-kill/concurrent-source integration, pre-inference pruning near - chunk limits, component-policy reuse and indexing/foreground measurements. +- [x] Pre-inference staging pruning with compatible rename remapping and + 100,000-chunk storage boundary; late validation after ANN preparation. +- [x] Exact parser-input SHA provenance and same-metadata overwrite detection + on normal rescans; stale/missing agent excerpts fail closed. +- [x] Actual Google CPU same-metadata text mutation before/during inference: + reject publication, rescan without metadata workarounds and reuse unaffected vectors. +- [ ] Process-kill/concurrent deletion/media mutation, component-policy reuse and + indexing/foreground measurements. See [INCREMENTAL-SEMANTIC.md](INCREMENTAL-SEMANTIC.md). - [ ] Benchmark genuine token-budgeted multi-sequence batching and isolation. - [ ] Implement and measure shared foreground/indexer resource budgets/priority.