Use isolated document extraction in shared indexing and desktop reconciliation

crates/searchi-core/tests/document_publication.rs

@@ -0,0 +1,107 @@
+//! Deterministic parent/index publication tests using an owned protocol fixture.
+//! Genuine parser execution is covered separately in documents.rs.
+use anyhow::Result;
+use searchi_core::{
+    extraction_protocol::{DocumentFormat, RequestHeader},
+    IndexOptions, SearchIndex,
+};
+use std::{
+    fs,
+    sync::atomic::{AtomicBool, Ordering},
+    time::{Duration, Instant},
+};
+
+#[test]
+fn mid_worker_same_stat_changes_and_cancellation_roll_back_index() -> Result<()> {
+    for cancel in [false, true] {
+        let directory = tempfile::tempdir()?;
+        let root = directory.path().join("files");
+        fs::create_dir(&root)?;
+        let source = root.join("contract.pdf");
+        fs::write(root.join("notes.txt"), "preserved evidence")?;
+        let mut index = SearchIndex::open(":memory:")?;
+        index.index_folder(&root)?;
+        fs::write(&source, b"original")?;
+        let initial = fs::metadata(&source)?;
+        let digest = RequestHeader::from_bytes(DocumentFormat::Pdf, b"original")?.input_sha256;
+        let marker = directory.path().join("admitted");
+        let release = directory.path().join("release");
+        let fixture_source = directory.path().join("worker.rs");
+        let executable = directory
+            .path()
+            .join(format!("fixture{}", std::env::consts::EXE_SUFFIX));
+        let response = serde_json::json!({
+            "version":1,"input_sha256":digest,
+            "outcome":{"status":"extracted","passages":[{"text":"stale document text","source":{"kind":"page","number":1}}]}
+        }).to_string();
+        fs::write(
+            &fixture_source,
+            format!(
+                r#"
+use std::io::{{Read,Write}};
+fn main() {{
+ let mut bytes=Vec::new(); std::io::stdin().read_to_end(&mut bytes).unwrap();
+ std::fs::write({marker:?}, b"admitted").unwrap();
+ while !std::path::Path::new({release:?}).exists() {{ std::thread::sleep(std::time::Duration::from_millis(5)); }}
+ let response={response:?}.as_bytes(); let mut out=std::io::stdout();
+ out.write_all(&(response.len() as u32).to_le_bytes()).unwrap(); out.write_all(response).unwrap();
+}}
+"#
+            ),
+        )?;
+        let status = std::process::Command::new("rustc")
+            .arg(&fixture_source)
+            .arg("-o")
+            .arg(&executable)
+            .status()?;
+        anyhow::ensure!(status.success(), "Compile owned protocol fixture");
+        index.set_document_worker(executable);
+        let cancelled = AtomicBool::new(false);
+        std::thread::scope(|scope| {
+            let changer = scope.spawn(|| -> Result<()> {
+                let deadline = Instant::now() + Duration::from_secs(10);
+                while !marker.exists() && Instant::now() < deadline {
+                    std::thread::sleep(Duration::from_millis(5));
+                }
+                if !marker.exists() {
+                    cancelled.store(true, Ordering::Release);
+                    anyhow::bail!("Worker did not admit fixture input");
+                }
+                if cancel {
+                    cancelled.store(true, Ordering::Release);
+                } else {
+                    fs::write(&source, b"modified")?;
+                    fs::File::options()
+                        .write(true)
+                        .open(&source)?
+                        .set_times(fs::FileTimes::new().set_modified(initial.modified()?))?;
+                    assert_eq!(fs::metadata(&source)?.len(), initial.len());
+                    assert_eq!(fs::metadata(&source)?.modified()?, initial.modified()?);
+                    fs::write(&release, b"release")?;
+                }
+                Ok(())
+            });
+            let result =
+                index.index_folder_cancellable(&root, &IndexOptions::default(), &cancelled);
+            assert!(
+                result.is_err(),
+                "Mutation or cancellation must roll back publication"
+            );
+            let message = format!("{:#}", result.unwrap_err());
+            assert!(
+                message.contains(if cancel {
+                    "cancelled"
+                } else {
+                    "changed during extraction"
+                }),
+                "{message}"
+            );
+            changer.join().unwrap().unwrap();
+        });
+        assert_eq!(index.stats()?.files, 1);
+        assert_eq!(index.search("evidence", 10)?.len(), 1);
+        assert!(index.search("stale", 10)?.is_empty());
+        assert!(!index.contains_path(&source.to_string_lossy())?);
+    }
+    Ok(())
+}

crates/searchi-core/tests/documents.rs

@@ -57,6 +57,7 @@ fn compressed_document_limits_are_enforced() -> Result<()> {
     );
     docx(&root.path().join("over-limit.docx"), &xml)?;
     let mut index = SearchIndex::open(":memory:")?;
+    index.use_in_process_document_parser_for_tests();
     assert_eq!(index.index_folder(root.path())?.failures, 2);
     assert_eq!(index.stats()?.content_files, 0);
     assert!(index.search("bomb.pdf", 10)?[0].extraction_error.is_some());
@@ -90,6 +91,7 @@ fn encrypted_pdf_is_explicitly_unsupported() -> Result<()> {
     document.encrypt(&state)?;
     document.save(&path)?;
     let mut index = SearchIndex::open(":memory:")?;
+    index.use_in_process_document_parser_for_tests();
     assert_eq!(index.index_folder(root.path())?.failures, 1);
     assert!(index.search("protected.pdf", 10)?[0]
         .extraction_error
@@ -122,6 +124,72 @@ fn docx(path: &Path, xml: &str) -> Result<()> {
     Ok(())
 }
 
+#[test]
+fn missing_worker_preserves_filename_clears_passages_and_retries() -> Result<()> {
+    let root = tempfile::tempdir()?;
+    pdf(&root.path().join("agreement.pdf"))?;
+    let mut index = SearchIndex::open(":memory:")?;
+    index.use_in_process_document_parser_for_tests();
+    index.index_folder(root.path())?;
+    assert!(!index.search("cancellation", 10)?.is_empty());
+    index.set_document_worker(root.path().join("missing-helper.exe"));
+    // Force a changed snapshot so a previous successful extraction is replaced.
+    pdf_with_text(
+        &root.path().join("agreement.pdf"),
+        &["different cancellation"],
+    )?;
+    assert_eq!(index.index_folder(root.path())?.failures, 1);
+    assert!(index.search("cancellation", 10)?.is_empty());
+    assert!(index.search("agreement.pdf", 10)?[0]
+        .extraction_error
+        .is_some());
+    // A same-stat, same-byte failed source must not be skipped on retry.
+    index.use_in_process_document_parser_for_tests();
+    assert_eq!(index.index_folder(root.path())?.indexed, 1);
+    assert!(!index.search("cancellation", 10)?.is_empty());
+    Ok(())
+}
+
+#[test]
+#[ignore = "requires built helper in SEARCHI_EXTRACT_WORKER"]
+fn normal_adapter_uses_real_document_worker_and_cancellation_rolls_back() -> Result<()> {
+    anyhow::ensure!(
+        std::env::var_os("SEARCHI_EXTRACT_WORKER").is_some(),
+        "Set actual document helper path"
+    );
+    let root = tempfile::tempdir()?;
+    pdf(&root.path().join("agreement.pdf"))?;
+    let mut index = SearchIndex::open(":memory:")?;
+    assert_eq!(index.index_folder(root.path())?.failures, 0);
+    assert_eq!(
+        index.search("cancellation", 10)?[0]
+            .source
+            .as_ref()
+            .unwrap()
+            .number,
+        2
+    );
+    let before = index.stats()?.files;
+    let cancelled = std::sync::atomic::AtomicBool::new(true);
+    assert!(index
+        .index_folder_cancellable(
+            root.path(),
+            &searchi_core::IndexOptions::default(),
+            &cancelled
+        )
+        .is_err());
+    assert_eq!(index.stats()?.files, before);
+    assert_eq!(
+        index.search("cancellation", 10)?[0]
+            .source
+            .as_ref()
+            .unwrap()
+            .number,
+        2
+    );
+    Ok(())
+}
+
 #[test]
 fn real_pdf_and_docx_return_matching_locations_and_persist() -> Result<()> {
     let root = tempfile::tempdir()?;
@@ -132,6 +200,7 @@ fn real_pdf_and_docx_return_matching_locations_and_persist() -> Result<()> {
     )?;
     let db = root.path().join("index.db");
     let mut index = SearchIndex::open(&db)?;
+    index.use_in_process_document_parser_for_tests();
     assert_eq!(index.index_folder(root.path())?.indexed, 2);
     let result = index.search("cancellation", 10)?;
     assert_eq!(result[0].source.as_ref().unwrap().kind, "page");
@@ -180,6 +249,7 @@ fn corrupt_document_reports_failure_and_preserves_filename_search() -> Result<()
     let root = tempfile::tempdir()?;
     fs::write(root.path().join("notes.txt"), "persistent evidence")?;
     let mut index = SearchIndex::open(":memory:")?;
+    index.use_in_process_document_parser_for_tests();
     index.index_folder(root.path())?;
     fs::write(root.path().join("broken.pdf"), b"not a PDF")?;
     assert_eq!(index.index_folder(root.path())?.failures, 1);
@@ -199,7 +269,8 @@ fn corrupt_document_reports_failure_and_preserves_filename_search() -> Result<()
     index.index_folder(root.path())?;
     assert_eq!(index.search("cancellation", 10)?.len(), 1);
     fs::write(root.path().join("agreement.pdf"), "corrupted replacement")?;
-    assert_eq!(index.index_folder(root.path())?.failures, 1);
+    // Previously rejected DOCX is retried too; failures are per current scan.
+    assert_eq!(index.index_folder(root.path())?.failures, 2);
     assert!(index.search("cancellation", 10)?.is_empty());
     assert!(index.search("agreement.pdf", 10)?[0]
         .extraction_error