Verify extraction provenance and prune semantic staging before inference

crates/searchi-service/src/pipeline.rs

@@ -53,6 +53,7 @@ impl HybridSearch {
     pub(super) fn rebuild_generation(
         &mut self,
         fingerprint: String,
+        media_limits: Option<&searchi_media::MediaLimits>,
         build: impl FnOnce(&mut Self, i64, &mut BuildWork) -> Result<()>,
     ) -> Result<EmbeddingReport> {
         let lock = self.store.build_lock()?;
@@ -83,28 +84,110 @@ impl HybridSearch {
             media: Vec::new(),
             documents: HashMap::new(),
         };
-        let result = build(self, generation, &mut work).and_then(|()| {
-            for identity in work.media.iter().chain(work.documents.values()) {
-                // Full bytes, not just stat metadata: in-place changes can
-                // preserve size and timestamps, including on mounted volumes.
-                let current = searchi_media::MediaIdentity::read(&identity.path, identity.bytes)?;
-                ensure!(
-                    current.sha256 == identity.sha256
-                        && current.modified_ns == identity.modified_ns,
-                    "Source changed during semantic indexing; retry required"
-                );
-            }
-            self.store
-                .reconcile_tasks(generation, &work.keys, revision)?;
-            self.store.activate_checked(generation, revision)?;
-            Ok(work.report)
-        });
+        let result = self
+            .discover_tasks(media_limits)
+            .and_then(|tasks| {
+                self.store
+                    .prune_before_inference(generation, &tasks, revision)?;
+                build(self, generation, &mut work)
+            })
+            .and_then(|()| {
+                self.store
+                    .reconcile_tasks(generation, &work.keys, revision)?;
+                self.store
+                    .activate_checked_with_validation(generation, revision, || {
+                        for identity in work.media.iter().chain(work.documents.values()) {
+                            // Full bytes, not just stat metadata: in-place changes can
+                            // preserve size and timestamps, including on mounted volumes.
+                            let current =
+                                searchi_media::MediaIdentity::read(&identity.path, identity.bytes)?;
+                            ensure!(
+                                current.sha256 == identity.sha256
+                                    && current.modified_ns == identity.modified_ns,
+                                "Source changed during semantic indexing; retry required"
+                            );
+                        }
+                        Ok(())
+                    })?;
+                Ok(work.report)
+            });
         if result.is_err() {
             self.generation_fingerprint = previous_fingerprint;
         }
         result
     }
 
+    /// Discover source identities in keyset batches before spending the encoder
+    /// budget. This extra byte-verification pass enables safe staging pruning;
+    /// it is indexing I/O, not a foreground latency optimization.
+    fn discover_tasks(
+        &self,
+        media_limits: Option<&searchi_media::MediaLimits>,
+    ) -> Result<Vec<SourceTask>> {
+        let mut tasks = Vec::new();
+        let mut after = 0;
+        loop {
+            let passages = self.lexical.passage_batch(after, 64)?;
+            if passages.is_empty() {
+                break;
+            }
+            for passage in passages {
+                after = passage.id;
+                if passage.text.trim().is_empty() {
+                    continue;
+                }
+                ensure!(
+                    tasks.len() < MAX_SCAN_CHUNKS,
+                    "Semantic source task budget exceeded"
+                );
+                tasks.push(text_task(&passage, &self.generation_fingerprint)?);
+            }
+        }
+        if let Some(limits) = media_limits {
+            let mut after = 0;
+            let mut count = 0;
+            loop {
+                let files = self.store.media_batch(after)?;
+                if files.is_empty() {
+                    break;
+                }
+                for (id, path, _, _) in files {
+                    after = id;
+                    let Some(kind) = searchi_media::classify(Path::new(&path)) else {
+                        continue;
+                    };
+                    count += 1;
+                    ensure!(
+                        count <= 10000,
+                        "Media generation exceeds 10000-file resource budget"
+                    );
+                    ensure!(
+                        tasks.len() < MAX_SCAN_CHUNKS,
+                        "Semantic source task budget exceeded"
+                    );
+                    let identity = searchi_media::MediaIdentity::read(
+                        Path::new(&path),
+                        if kind == searchi_media::MediaKind::Image {
+                            64 * 1024 * 1024
+                        } else {
+                            limits.max_file_bytes
+                        },
+                    );
+                    tasks.push(SourceTask {
+                        key: task_key(&["media", &path]),
+                        path,
+                        source_type: "media".into(),
+                        source_hash: identity.map(|identity| identity.sha256).unwrap_or_default(),
+                        title_hash: String::new(),
+                        policy: self.generation_fingerprint.clone(),
+                        passage_id: -id,
+                    });
+                }
+            }
+        }
+        Ok(tasks)
+    }
+
     pub(super) fn build(&mut self, generation: i64, work: &mut BuildWork) -> Result<()> {
         let mut after_id = 0;
         loop {
@@ -126,6 +209,9 @@ impl HybridSearch {
                         Path::new(&passage.path),
                         32 * 1024 * 1024,
                     )?;
+                    let extracted = self.lexical.extracted_source_hash(&passage.path)?;
+                    ensure!(extracted.as_deref() == Some(identity.sha256.as_str()),
+                        "Document bytes differ from extracted passages; reindex the folder before semantic indexing");
                     work.documents.insert(passage.path.clone(), identity);
                 }
                 let task = text_task(&passage, &self.generation_fingerprint)?;