Reuse completed Google embeddings and resume semantic staging

crates/searchi-service/src/store/incremental.rs

@@ -0,0 +1,671 @@
+//! Completed-source manifests and resumable staging. Legacy generations stay
+//! searchable but cannot supply vectors without explicit completion evidence.
+use super::*;
+
+#[derive(Debug, Clone)]
+pub(crate) struct SourceTask {
+    pub key: String,
+    pub path: String,
+    /// `text` for one extracted passage, `media` for the entire media file.
+    pub source_type: String,
+    pub source_hash: String,
+    pub title_hash: String,
+    pub policy: String,
+    pub passage_id: i64,
+}
+
+pub(super) fn migrate(connection: &Connection) -> Result<()> {
+    for (table, name, declaration) in [
+        (
+            "semantic_generations",
+            "incremental",
+            "INTEGER NOT NULL DEFAULT 0",
+        ),
+        ("semantic_chunks", "task_key", "TEXT"),
+    ] {
+        let exists: bool = connection.query_row(
+            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)",
+            params![table, name],
+            |r| r.get(0),
+        )?;
+        if !exists {
+            connection.execute_batch(&format!(
+                "ALTER TABLE {table} ADD COLUMN {name} {declaration}"
+            ))?;
+        }
+    }
+    connection.execute_batch("CREATE TABLE IF NOT EXISTS semantic_tasks(
+        generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,
+        task_key TEXT NOT NULL,path TEXT NOT NULL,source_type TEXT NOT NULL,
+        source_hash TEXT NOT NULL,title_hash TEXT NOT NULL,policy TEXT NOT NULL,
+        passage_id INTEGER NOT NULL,state TEXT NOT NULL CHECK(state IN ('pending','running','done','failed')),
+        attempts INTEGER NOT NULL DEFAULT 0,expected_chunks INTEGER,message TEXT,
+        PRIMARY KEY(generation,task_key));
+        CREATE INDEX IF NOT EXISTS semantic_task_reuse ON semantic_tasks(source_hash,title_hash,policy,source_type,state);
+        CREATE INDEX IF NOT EXISTS semantic_chunk_task ON semantic_chunks(generation,task_key);
+        UPDATE semantic_meta SET schema_version=7 WHERE singleton=1;")?;
+    Ok(())
+}
+
+pub(super) fn validate_completion(connection: &Connection, generation: i64) -> Result<()> {
+    let incomplete: bool = connection.query_row("SELECT EXISTS(SELECT 1 FROM semantic_tasks t WHERE generation=?1 AND
+        (state NOT IN ('done','failed') OR (state='done' AND (expected_chunks IS NULL OR expected_chunks!=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key))) OR
+        (state='failed' AND EXISTS(SELECT 1 FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key))))",[generation], |r|r.get(0))?;
+    ensure!(
+        !incomplete,
+        "Semantic generation contains incomplete source tasks"
+    );
+    let unowned: bool = connection.query_row("SELECT EXISTS(SELECT 1 FROM semantic_chunks c LEFT JOIN semantic_tasks t ON t.generation=c.generation AND t.task_key=c.task_key WHERE c.generation=?1 AND (t.state IS NULL OR t.state!='done'))",[generation],|r|r.get(0))?;
+    ensure!(
+        !unowned,
+        "Semantic generation contains chunks without completed sources"
+    );
+    Ok(())
+}
+
+impl Store {
+    pub(crate) fn revision(&self) -> Result<i64> {
+        Ok(self.connection.query_row(
+            "SELECT corpus_revision FROM semantic_meta WHERE singleton=1",
+            [],
+            |r| r.get(0),
+        )?)
+    }
+
+    /// Caller must retain the OS lock through reconciliation and publication.
+    pub(crate) fn start_or_resume(
+        &mut self,
+        fingerprint: &str,
+        lock: &BuildLock,
+    ) -> Result<(i64, bool)> {
+        ensure!(
+            lock.1 == self.lock_path,
+            "Build lock belongs to a different index"
+        );
+        ensure!(
+            !fingerprint.is_empty() && fingerprint.len() <= 4096,
+            "Invalid generation fingerprint"
+        );
+        let tx = self
+            .connection
+            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        let generation: Option<i64>=tx.query_row("SELECT id FROM semantic_generations WHERE id!=COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1) AND fingerprint=?1 AND dimensions=?2 AND incremental=1 ORDER BY id DESC LIMIT 1",params![fingerprint,self.dimensions as i64],|r|r.get(0)).optional()?;
+        let resumed = generation.is_some();
+        let id = if let Some(id) = generation {
+            id
+        } else {
+            tx.execute("INSERT INTO semantic_generations(fingerprint,dimensions,corpus_revision,incremental) SELECT ?1,?2,corpus_revision,1 FROM semantic_meta WHERE singleton=1",params![fingerprint,self.dimensions as i64])?;
+            tx.last_insert_rowid()
+        };
+        tx.execute("DELETE FROM semantic_generations WHERE id!=?1 AND id!=COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1)",[id])?;
+        // Untagged chunks are writes interrupted before the source transaction.
+        tx.execute("UPDATE semantic_tasks SET state='pending',expected_chunks=NULL WHERE generation=?1 AND state='done' AND
+            (expected_chunks IS NULL OR expected_chunks!=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=semantic_tasks.generation AND c.task_key=semantic_tasks.task_key))",[id])?;
+        tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND NOT EXISTS(SELECT 1 FROM semantic_tasks t WHERE t.generation=semantic_chunks.generation AND t.task_key=semantic_chunks.task_key AND t.state='done')",[id])?;
+        tx.execute("UPDATE semantic_tasks SET state='pending',expected_chunks=NULL,message=NULL WHERE generation=?1 AND state!='done'",[id])?;
+        tx.commit()?;
+        Ok((id, resumed))
+    }
+
+    pub(crate) fn stage_task(&mut self, generation: i64, task: &SourceTask) -> Result<bool> {
+        validate_task(task)?;
+        let tx = self.connection.transaction()?;
+        let same:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM semantic_tasks WHERE generation=?1 AND task_key=?2 AND source_hash=?3 AND title_hash=?4 AND policy=?5 AND source_type=?6 AND state='done')",params![generation,task.key,task.source_hash,task.title_hash,task.policy,task.source_type],|r|r.get(0))?;
+        if !same {
+            tx.execute(
+                "DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2",
+                params![generation, task.key],
+            )?;
+            tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts)
+                VALUES(?1,?2,?3,?4,?5,?6,?7,?8,'running',1)
+                ON CONFLICT(generation,task_key) DO UPDATE SET path=excluded.path,source_type=excluded.source_type,source_hash=excluded.source_hash,title_hash=excluded.title_hash,policy=excluded.policy,passage_id=excluded.passage_id,state='running',attempts=semantic_tasks.attempts+1,expected_chunks=NULL,message=NULL",params![generation,task.key,task.path,task.source_type,task.source_hash,task.title_hash,task.policy,task.passage_id])?;
+        }
+        tx.commit()?;
+        Ok(same)
+    }
+
+    pub(crate) fn complete_source(
+        &mut self,
+        generation: i64,
+        task: &SourceTask,
+        expected_chunks: usize,
+    ) -> Result<()> {
+        ensure!(
+            !task.source_hash.is_empty(),
+            "Cannot complete a source without verified identity"
+        );
+        ensure!(
+            expected_chunks <= MAX_SCAN_CHUNKS,
+            "Source exceeds semantic chunk budget"
+        );
+        let tx = self
+            .connection
+            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        let count:i64=tx.query_row("SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type!='text'))",params![generation,task.path,task.source_type,task.passage_id],|r|r.get(0))?;
+        ensure!(
+            usize::try_from(count)? == expected_chunks,
+            "Incomplete source chunk membership"
+        );
+        let total: usize = tx.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [generation],
+            |r| r.get(0),
+        )?;
+        ensure!(
+            total <= MAX_SCAN_CHUNKS,
+            "Generation exceeds semantic chunk budget"
+        );
+        tx.execute("UPDATE semantic_chunks SET task_key=?5 WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type!='text'))",params![generation,task.path,task.source_type,task.passage_id,task.key])?;
+        let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?;
+        ensure!(changed == 1, "Missing matching source task");
+        if task.source_type == "media" {
+            tx.execute(
+                "DELETE FROM semantic_media_errors WHERE generation=?1 AND path=?2",
+                params![generation, task.path],
+            )?;
+        }
+        tx.commit()?;
+        Ok(())
+    }
+
+    pub(crate) fn fail_source(
+        &mut self,
+        generation: i64,
+        task: &SourceTask,
+        message: &str,
+    ) -> Result<()> {
+        let tx = self.connection.transaction()?;
+        tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND (task_key=?2 OR (path=?3 AND ((?4='text' AND source_type='text' AND passage_id=?5) OR (?4='media' AND source_type!='text'))))",params![generation,task.key,task.path,task.source_type,task.passage_id])?;
+        tx.execute("UPDATE semantic_tasks SET state='failed',expected_chunks=NULL,message=?3 WHERE generation=?1 AND task_key=?2",params![generation,task.key,message.chars().take(2048).collect::<String>()])?;
+        tx.commit()?;
+        Ok(())
+    }
+
+    pub(crate) fn reconcile_tasks(
+        &mut self,
+        generation: i64,
+        keys: &[String],
+        revision: i64,
+    ) -> Result<()> {
+        ensure!(
+            keys.len() <= MAX_SCAN_CHUNKS,
+            "Generation exceeds source task budget"
+        );
+        let tx = self
+            .connection
+            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        let current: i64 = tx.query_row(
+            "SELECT corpus_revision FROM semantic_meta WHERE singleton=1",
+            [],
+            |r| r.get(0),
+        )?;
+        ensure!(
+            current == revision,
+            "Indexed corpus changed during semantic reconciliation; retry required"
+        );
+        tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_current_tasks(task_key TEXT PRIMARY KEY); DELETE FROM semantic_current_tasks;")?;
+        for key in keys {
+            tx.execute(
+                "INSERT OR IGNORE INTO semantic_current_tasks VALUES(?1)",
+                [key],
+            )?;
+        }
+        tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND (task_key IS NULL OR task_key NOT IN (SELECT task_key FROM semantic_current_tasks))",[generation])?;
+        tx.execute("DELETE FROM semantic_tasks WHERE generation=?1 AND task_key NOT IN (SELECT task_key FROM semantic_current_tasks)",[generation])?;
+        tx.execute("DELETE FROM semantic_media_errors WHERE generation=?1 AND path NOT IN (SELECT path FROM semantic_tasks WHERE generation=?1)",[generation])?;
+        tx.execute(
+            "UPDATE semantic_generations SET corpus_revision=?2 WHERE id=?1",
+            params![generation, revision],
+        )?;
+        tx.commit()?;
+        Ok(())
+    }
+
+    pub(crate) fn activate_checked(&mut self, generation: i64, revision: i64) -> Result<()> {
+        validate_completion(&self.connection, generation)?;
+        self.activate_revision(generation, Some(revision))
+    }
+}
+
+fn validate_task(task: &SourceTask) -> Result<()> {
+    ensure!(
+        !task.key.is_empty()
+            && task.key.len() <= 256
+            && !task.path.is_empty()
+            && task.path.len() <= 32768,
+        "Invalid source task identity"
+    );
+    ensure!(
+        matches!(task.source_type.as_str(), "text" | "media"),
+        "Invalid source task type"
+    );
+    ensure!(
+        (!task.source_hash.is_empty() || task.source_type == "media")
+            && task.source_hash.len() <= 256
+            && task.title_hash.len() <= 256
+            && !task.policy.is_empty()
+            && task.policy.len() <= 4096,
+        "Invalid source completion policy"
+    );
+    Ok(())
+}
+
+impl Store {
+    fn reusable_task(
+        &self,
+        generation: i64,
+        task: &SourceTask,
+    ) -> Result<Option<(i64, String, usize)>> {
+        Ok(self.connection.query_row("SELECT t.generation,t.task_key,t.expected_chunks FROM semantic_tasks t JOIN semantic_generations g ON g.id=t.generation JOIN semantic_generations destination ON destination.id=?1
+            WHERE t.source_hash=?2 AND t.title_hash=?3 AND t.policy=?4 AND t.source_type=?5 AND t.state='done' AND t.expected_chunks IS NOT NULL
+            AND g.fingerprint=destination.fingerprint AND g.dimensions=destination.dimensions
+            AND t.expected_chunks=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key)
+            ORDER BY (t.generation=?1) DESC,t.generation DESC LIMIT 1",params![generation,task.source_hash,task.title_hash,task.policy,task.source_type],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?)
+    }
+
+    /// No tokenizer or neural call: only a completed, policy-compatible group
+    /// whose persisted ranges still equal the current extracted bytes qualifies.
+    pub(crate) fn reuse_text_source(
+        &mut self,
+        generation: i64,
+        task: &SourceTask,
+        passage: &IndexedPassage,
+    ) -> Result<Option<usize>> {
+        ensure!(
+            task.source_type == "text"
+                && task.passage_id == passage.id
+                && task.path == passage.path,
+            "Text task mapping differs from current source"
+        );
+        ensure!(
+            task.source_hash == content_hash(&passage.text),
+            "Text task hash differs from source bytes"
+        );
+        let Some((source, key, count)) = self.reusable_task(generation, task)? else {
+            return Ok(None);
+        };
+        ensure!(
+            count <= MAX_SCAN_CHUNKS,
+            "Reusable source exceeds chunk budget"
+        );
+        {
+            let mut stmt=self.connection.prepare("SELECT start_byte,end_byte,snippet,vector,source_type,ordinal,content_hash FROM semantic_chunks WHERE generation=?1 AND task_key=?2 ORDER BY ordinal")?;
+            let mut rows = stmt.query(params![source, key])?;
+            let mut seen = 0usize;
+            while let Some(row) = rows.next()? {
+                let start: usize = row.get(0)?;
+                let end: usize = row.get(1)?;
+                let snippet: String = row.get(2)?;
+                ensure!(
+                    passage.text.get(start..end) == Some(snippet.as_str()),
+                    "Reusable text range differs from current source"
+                );
+                ensure!(
+                    row.get::<_, String>(4)? == "text"
+                        && row.get::<_, usize>(5)? == seen
+                        && row.get::<_, String>(6)? == task.source_hash,
+                    "Corrupt completed text source membership"
+                );
+                validate_vector(&row.get::<_, Vec<u8>>(3)?, self.dimensions)?;
+                seen += 1;
+            }
+            ensure!(seen == count, "Reusable text completion count differs");
+        }
+        self.copy_completed(
+            generation,
+            task,
+            source,
+            &key,
+            count,
+            Some(&passage.source),
+            None,
+        )?;
+        Ok(Some(count))
+    }
+
+    pub(crate) fn reuse_media_source(
+        &mut self,
+        generation: i64,
+        task: &SourceTask,
+        file_id: i64,
+        identity: &searchi_media::MediaIdentity,
+    ) -> Result<Option<usize>> {
+        ensure!(
+            task.source_type == "media"
+                && task.passage_id == -file_id
+                && task.path == identity.path.to_string_lossy()
+                && task.source_hash == identity.sha256,
+            "Media task mapping differs from current source"
+        );
+        let Some((source, key, count)) = self.reusable_task(generation, task)? else {
+            return Ok(None);
+        };
+        ensure!(
+            count <= MAX_SCAN_CHUNKS,
+            "Reusable source exceeds chunk budget"
+        );
+        {
+            let mut stmt=self.connection.prepare("SELECT vector,source_type,ordinal,content_hash FROM semantic_chunks WHERE generation=?1 AND task_key=?2 ORDER BY ordinal")?;
+            let mut rows = stmt.query(params![source, key])?;
+            let mut seen = 0usize;
+            while let Some(row) = rows.next()? {
+                ensure!(
+                    matches!(
+                        row.get::<_, String>(1)?.as_str(),
+                        "image" | "audio" | "video"
+                    ) && row.get::<_, usize>(2)? == seen
+                        && row.get::<_, String>(3)? == task.source_hash,
+                    "Corrupt completed media source membership"
+                );
+                validate_vector(&row.get::<_, Vec<u8>>(0)?, self.dimensions)?;
+                seen += 1;
+            }
+            ensure!(seen == count, "Reusable media completion count differs");
+        }
+        self.copy_completed(generation, task, source, &key, count, None, Some(identity))?;
+        Ok(Some(count))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn copy_completed(
+        &mut self,
+        generation: i64,
+        task: &SourceTask,
+        source: i64,
+        key: &str,
+        count: usize,
+        location: Option<&SourceLocation>,
+        identity: Option<&searchi_media::MediaIdentity>,
+    ) -> Result<()> {
+        let tx = self
+            .connection
+            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        let total:usize=tx.query_row("SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND (task_key IS NULL OR task_key!=?2)",params![generation,task.key],|r|r.get(0))?;
+        ensure!(
+            total.saturating_add(count) <= MAX_SCAN_CHUNKS,
+            "Generation exceeds semantic chunk budget"
+        );
+        // A temporary SQL table allows remapping a completed staging task onto
+        // itself without retaining its old citations or allocating all vectors.
+        tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_reuse_chunks AS SELECT * FROM semantic_chunks WHERE 0; DELETE FROM semantic_reuse_chunks;")?;
+        tx.execute("INSERT INTO semantic_reuse_chunks SELECT * FROM semantic_chunks WHERE generation=?1 AND task_key=?2",params![source,key])?;
+        tx.execute(
+            "DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2",
+            params![generation, task.key],
+        )?;
+        tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
+            SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8 FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
+        let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count as i64,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?;
+        ensure!(changed == 1, "Missing source task during reuse");
+        if task.source_type == "media" {
+            tx.execute(
+                "DELETE FROM semantic_media_errors WHERE generation=?1 AND path=?2",
+                params![generation, task.path],
+            )?;
+        }
+        tx.execute_batch("DELETE FROM semantic_reuse_chunks;")?;
+        tx.commit()?;
+        Ok(())
+    }
+}
+
+fn validate_vector(bytes: &[u8], dimensions: usize) -> Result<()> {
+    ensure!(
+        bytes.len() == dimensions * 4,
+        "Corrupt reusable vector length"
+    );
+    let mut norm = 0.0f32;
+    for value in bytes.as_chunks::<4>().0 {
+        let value = f32::from_le_bytes(*value);
+        ensure!(value.is_finite(), "Corrupt reusable vector value");
+        norm += value * value;
+    }
+    ensure!(
+        norm.is_finite() && (norm.sqrt() - 1.0).abs() < 1e-3,
+        "Reusable vector is not normalized"
+    );
+    Ok(())
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use searchi_core::SearchIndex;
+
+    // Storage-only vectors exercise durability and membership. They are not
+    // simulated production inference or evidence of semantic relevance.
+    fn vector() -> Vec<f32> {
+        vec![1.0, 0.0]
+    }
+    fn task(p: &IndexedPassage) -> SourceTask {
+        SourceTask {
+            key: format!("text-{}-{}", p.path, p.source.number),
+            path: p.path.clone(),
+            source_type: "text".into(),
+            source_hash: content_hash(&p.text),
+            title_hash: content_hash(&p.filename),
+            policy: "storage-test-window-policy".into(),
+            passage_id: p.id,
+        }
+    }
+    fn insert_complete(
+        store: &mut Store,
+        g: i64,
+        p: &IndexedPassage,
+        t: &SourceTask,
+    ) -> Result<()> {
+        store.stage_task(g, t)?;
+        store.insert(
+            g,
+            p,
+            0,
+            &[TextWindow {
+                text: p.text.clone(),
+                start_byte: 0,
+                end_byte: p.text.len(),
+            }],
+            &[vector()],
+        )?;
+        store.complete_source(g, t, 1)
+    }
+
+    #[test]
+    fn resume_retains_completed_groups_and_discards_unfinished_vectors() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let root = dir.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("first.txt"), "complete source")?;
+        std::fs::write(root.join("second.txt"), "interrupted source")?;
+        let db = dir.path().join("index.db");
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let passages = lexical.passage_batch(0, 64)?;
+        let first = task(&passages[0]);
+        let second = task(&passages[1]);
+        let generation;
+        {
+            let mut store = Store::open(&db, 2)?;
+            let lock = store.build_lock()?;
+            generation = store.start_or_resume("storage-only-model", &lock)?.0;
+            insert_complete(&mut store, generation, &passages[0], &first)?;
+            store.stage_task(generation, &second)?;
+            store.insert(
+                generation,
+                &passages[1],
+                0,
+                &[TextWindow {
+                    text: passages[1].text.clone(),
+                    start_byte: 0,
+                    end_byte: passages[1].text.len(),
+                }],
+                &[vector()],
+            )?;
+            assert!(store
+                .activate_checked(generation, store.revision()?)
+                .is_err());
+        }
+        let mut store = Store::open(&db, 2)?;
+        let lock = store.build_lock()?;
+        assert_eq!(
+            store.start_or_resume("storage-only-model", &lock)?,
+            (generation, true)
+        );
+        assert_eq!(
+            store.connection.query_row(
+                "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+                [generation],
+                |r| r.get::<_, usize>(0)
+            )?,
+            1
+        );
+        store.stage_task(generation, &first)?;
+        assert_eq!(
+            store.reuse_text_source(generation, &first, &passages[0])?,
+            Some(1)
+        );
+        assert!(store
+            .reuse_text_source(generation, &second, &passages[1])?
+            .is_none());
+        assert!(store
+            .activate_checked(generation, store.revision()?)
+            .is_err());
+        store.reconcile_tasks(generation, &[first.key], store.revision()?)?;
+        store.activate_checked(generation, store.revision()?)?;
+        Ok(())
+    }
+
+    #[test]
+    fn legacy_vectors_remain_searchable_but_do_not_qualify_for_reuse() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let root = dir.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "legacy searchable source")?;
+        let db = dir.path().join("index.db");
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let p = lexical.passage_batch(0, 1)?.remove(0);
+        let mut store = Store::open(&db, 2)?;
+        let legacy = store.begin("legacy-policy")?;
+        store.insert(
+            legacy,
+            &p,
+            0,
+            &[TextWindow {
+                text: p.text.clone(),
+                start_byte: 0,
+                end_byte: p.text.len(),
+            }],
+            &[vector()],
+        )?;
+        store.activate(legacy)?;
+        // Reopen a schema6 DB with legacy active chunks and no manifests.
+        store.connection.execute_batch(
+            "DROP INDEX semantic_chunk_task; ALTER TABLE semantic_chunks DROP COLUMN task_key;
+            DROP TABLE semantic_tasks; ALTER TABLE semantic_generations DROP COLUMN incremental;
+            UPDATE semantic_meta SET schema_version=6;",
+        )?;
+        drop(store);
+        let mut store = Store::open(&db, 2)?;
+        assert_eq!(
+            store
+                .search(&vector(), "legacy-policy", 1, &SearchOptions::default())?
+                .len(),
+            1
+        );
+        let lock = store.build_lock()?;
+        let (g, _) = store.start_or_resume("legacy-policy", &lock)?;
+        let t = task(&p);
+        store.stage_task(g, &t)?;
+        assert_eq!(store.reuse_text_source(g, &t, &p)?, None);
+        assert_eq!(
+            store.active_fingerprint()?.as_deref(),
+            Some("legacy-policy")
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn revision_race_cannot_publish_and_failed_activation_keeps_staging() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let root = dir.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "source before change")?;
+        let db = dir.path().join("index.db");
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let p = lexical.passage_batch(0, 1)?.remove(0);
+        let mut store = Store::open(&db, 2)?;
+        let lock = store.build_lock()?;
+        let (g, _) = store.start_or_resume("checked-policy", &lock)?;
+        let t = task(&p);
+        insert_complete(&mut store, g, &p, &t)?;
+        let old_revision = store.revision()?;
+        std::fs::write(root.join("notes.txt"), "source after change longer")?;
+        lexical.index_folder(&root)?;
+        assert!(store.activate_checked(g, old_revision).is_err());
+        assert_eq!(store.active_fingerprint()?, None);
+        assert_eq!(store.start_or_resume("checked-policy", &lock)?, (g, true));
+        assert_eq!(
+            store.connection.query_row(
+                "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+                [g],
+                |r| r.get::<_, usize>(0)
+            )?,
+            1
+        );
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod completion_failure_tests {
+    use super::*;
+    use searchi_core::SearchIndex;
+
+    #[test]
+    fn failed_completion_rolls_back_chunk_membership_and_malformed_ranges_never_reuse() -> Result<()>
+    {
+        // SQLite durability fixture, not neural inference evidence.
+        let dir = tempfile::tempdir()?;
+        let root = dir.path().join("docs");
+        std::fs::create_dir(&root)?;
+        std::fs::write(root.join("notes.txt"), "UTF-8 source with 中文 preserved")?;
+        let db = dir.path().join("index.db");
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let p = lexical.passage_batch(0, 1)?.remove(0);
+        let t = SourceTask {
+            key: "durability-task".into(),
+            path: p.path.clone(),
+            source_type: "text".into(),
+            source_hash: content_hash(&p.text),
+            title_hash: content_hash(&p.filename),
+            policy: "exact-window-test".into(),
+            passage_id: p.id,
+        };
+        let mut store = Store::open(&db, 2)?;
+        let lock = store.build_lock()?;
+        let (g, _) = store.start_or_resume("storage-geometry", &lock)?;
+        store.stage_task(g, &t)?;
+        store.insert(
+            g,
+            &p,
+            0,
+            &[TextWindow {
+                text: p.text.clone(),
+                start_byte: 0,
+                end_byte: p.text.len(),
+            }],
+            &[vec![1.0, 0.0]],
+        )?;
+        assert!(store.complete_source(g, &t, 2).is_err());
+        let (state,owned):(String,usize)=store.connection.query_row("SELECT state,(SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND task_key IS NOT NULL) FROM semantic_tasks WHERE generation=?1 AND task_key=?2",params![g,t.key],|r|Ok((r.get(0)?,r.get(1)?)))?;
+        assert_eq!(state, "running");
+        assert_eq!(owned, 0);
+        assert!(store.activate_checked(g, store.revision()?).is_err());
+        store.complete_source(g, &t, 1)?;
+        store.connection.execute("UPDATE semantic_chunks SET start_byte=1,snippet='different snippet' WHERE generation=?1",[g])?;
+        assert!(store.reuse_text_source(g, &t, &p).is_err());
+        assert!(store.active_fingerprint()?.is_none());
+        Ok(())
+    }
+}