feat: integrate Google semantic search and branded desktop launcher

crates/searchi-service/src/store.rs

@@ -0,0 +1,855 @@
+use std::{
+    collections::{HashMap, HashSet},
+    fs::{File, OpenOptions},
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use anyhow::{ensure, Context, Result};
+use fs2::FileExt;
+use rusqlite::{params, Connection, OptionalExtension};
+use searchi_core::{IndexedPassage, SearchOptions, SearchResult, SourceLocation};
+use searchi_semantic::TextWindow;
+use sha2::{Digest, Sha256};
+use usearch::{Index, IndexOptions, MetricKind, ScalarKind};
+
+pub(crate) const MAX_SCAN_CHUNKS: usize = 100_000;
+pub(crate) const MIN_COSINE: f32 = 0.25;
+
+#[derive(Clone, PartialEq, Eq)]
+struct FilterKey {
+    folder: Option<String>,
+    extension: Option<String>,
+    min_size: Option<i64>,
+    max_size: Option<i64>,
+    modified_after: Option<i64>,
+    modified_before: Option<i64>,
+}
+
+struct EligibilityCache {
+    generation: i64,
+    revision: i64,
+    filter: FilterKey,
+    keys: Arc<HashSet<u64>>,
+    by_path: HashMap<String, Vec<u64>>,
+}
+
+pub(crate) struct Store {
+    connection: Connection,
+    lock_path: PathBuf,
+    dimensions: usize,
+    ann_cache: Option<(i64, Index)>,
+    eligibility_cache: Option<EligibilityCache>,
+}
+
+pub(crate) struct BuildLock(File);
+impl Drop for BuildLock {
+    fn drop(&mut self) {
+        let _ = FileExt::unlock(&self.0);
+    }
+}
+
+pub(crate) fn content_hash(text: &str) -> String {
+    #[cfg(test)]
+    HASH_CALLS.with(|count| count.set(count.get() + 1));
+    format!("{:x}", Sha256::digest(text.as_bytes()))
+}
+
+#[cfg(test)]
+thread_local! { static HASH_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
+
+impl Store {
+    pub(crate) fn open(db: &Path, dimensions: usize) -> Result<Self> {
+        ensure!(
+            (1..=4096).contains(&dimensions),
+            "Embedding dimensions must be between 1 and 4096"
+        );
+        ensure!(
+            db != Path::new(":memory:"),
+            "Hybrid search requires a persistent database path"
+        );
+        let mut connection = Connection::open(db)?;
+        connection.busy_timeout(std::time::Duration::from_secs(5))?;
+        connection.execute_batch("PRAGMA foreign_keys=ON;")?;
+        let initialization =
+            connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+        initialization.execute_batch(
+            "CREATE TABLE IF NOT EXISTS semantic_meta (
+            singleton INTEGER PRIMARY KEY CHECK(singleton=1), schema_version INTEGER NOT NULL,
+            active_generation INTEGER, corpus_revision INTEGER NOT NULL DEFAULT 0, dirty_floor INTEGER NOT NULL DEFAULT 0);
+            INSERT OR IGNORE INTO semantic_meta(singleton,schema_version) VALUES(1,5);",
+        )?;
+        let version: i64 = initialization.query_row(
+            "SELECT schema_version FROM semantic_meta WHERE singleton=1",
+            [],
+            |row| row.get(0),
+        )?;
+        ensure!(
+            (1..=5).contains(&version),
+            "Unsupported semantic schema version {version}"
+        );
+        initialization.execute_batch("
+            CREATE TABLE IF NOT EXISTS semantic_generations(id INTEGER PRIMARY KEY, fingerprint TEXT NOT NULL, dimensions INTEGER NOT NULL, corpus_revision INTEGER NOT NULL DEFAULT 0, ann_blob BLOB, ann_sha256 TEXT);
+            CREATE TABLE IF NOT EXISTS semantic_chunks(
+                generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,
+                passage_id INTEGER NOT NULL, ordinal INTEGER NOT NULL,
+                path TEXT NOT NULL, kind TEXT NOT NULL, number INTEGER NOT NULL,
+                content_hash TEXT NOT NULL, start_byte INTEGER NOT NULL, end_byte INTEGER NOT NULL,
+                snippet TEXT NOT NULL, vector BLOB NOT NULL,
+                PRIMARY KEY(generation,passage_id,ordinal));")?;
+        if version < 5 {
+            for (table,column,alter) in [
+                ("semantic_meta","corpus_revision","ALTER TABLE semantic_meta ADD COLUMN corpus_revision INTEGER NOT NULL DEFAULT 0"),
+                ("semantic_meta","dirty_floor","ALTER TABLE semantic_meta ADD COLUMN dirty_floor INTEGER NOT NULL DEFAULT 0"),
+                ("semantic_generations","dimensions","ALTER TABLE semantic_generations ADD COLUMN dimensions INTEGER NOT NULL DEFAULT 384"),
+                ("semantic_generations","corpus_revision","ALTER TABLE semantic_generations ADD COLUMN corpus_revision INTEGER NOT NULL DEFAULT 0"),
+                ("semantic_generations","ann_blob","ALTER TABLE semantic_generations ADD COLUMN ann_blob BLOB"),
+                ("semantic_generations","ann_sha256","ALTER TABLE semantic_generations ADD COLUMN ann_sha256 TEXT"),
+            ] {
+                let exists:bool=initialization.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)",params![table,column],|row| row.get(0))?;
+                if !exists { initialization.execute_batch(alter)?; }
+            }
+            initialization.execute_batch("DROP TRIGGER IF EXISTS semantic_files_insert;
+                DROP TRIGGER IF EXISTS semantic_files_update; DROP TRIGGER IF EXISTS semantic_files_delete;")?;
+        }
+        initialization.execute_batch(
+            "CREATE TABLE IF NOT EXISTS semantic_dirty_paths(path TEXT PRIMARY KEY, revision INTEGER NOT NULL);
+            CREATE INDEX IF NOT EXISTS semantic_dirty_revision ON semantic_dirty_paths(revision);
+            CREATE INDEX IF NOT EXISTS semantic_chunk_path ON semantic_chunks(generation,path);
+            CREATE TRIGGER IF NOT EXISTS semantic_files_insert AFTER INSERT ON files BEGIN
+            UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
+            INSERT INTO semantic_dirty_paths(path,revision) VALUES(new.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
+                ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;
+            CREATE TRIGGER IF NOT EXISTS semantic_files_update AFTER UPDATE ON files BEGIN
+            UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
+            INSERT INTO semantic_dirty_paths(path,revision) VALUES(old.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
+                ON CONFLICT(path) DO UPDATE SET revision=excluded.revision;
+            INSERT INTO semantic_dirty_paths(path,revision) VALUES(new.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
+                ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;
+            CREATE TRIGGER IF NOT EXISTS semantic_files_delete AFTER DELETE ON files BEGIN
+            UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
+            INSERT INTO semantic_dirty_paths(path,revision) VALUES(old.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
+                ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;
+            UPDATE semantic_meta SET schema_version=5 WHERE singleton=1;",
+        )?;
+        initialization.commit()?;
+        let canonical = db.canonicalize()?;
+        let lock_path = PathBuf::from(format!("{}.semantic.lock", canonical.to_string_lossy()));
+        Ok(Self {
+            connection,
+            lock_path,
+            dimensions,
+            ann_cache: None,
+            eligibility_cache: None,
+        })
+    }
+
+    pub(crate) fn build_lock(&self) -> Result<BuildLock> {
+        let file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .open(&self.lock_path)?;
+        file.try_lock_exclusive()
+            .context("Another semantic rebuild is running")?;
+        Ok(BuildLock(file))
+    }
+
+    pub(crate) fn begin(&mut self, fingerprint: &str) -> Result<i64> {
+        let transaction = self.connection.transaction()?;
+        // Only called while holding the cross-process lock: abandoned staging
+        // generations may be removed safely after a crash.
+        transaction.execute("DELETE FROM semantic_generations WHERE id != COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1)", [])?;
+        transaction.execute(
+            "INSERT INTO semantic_generations(fingerprint,dimensions,corpus_revision) SELECT ?1,?2,corpus_revision FROM semantic_meta WHERE singleton=1",
+            params![fingerprint, self.dimensions as i64],
+        )?;
+        let id = transaction.last_insert_rowid();
+        transaction.commit()?;
+        Ok(id)
+    }
+
+    pub(crate) fn insert(
+        &mut self,
+        generation: i64,
+        passage: &IndexedPassage,
+        first_ordinal: usize,
+        windows: &[TextWindow],
+        vectors: &[Vec<f32>],
+    ) -> Result<()> {
+        ensure!(
+            windows.len() == vectors.len(),
+            "Window and embedding counts differ"
+        );
+        let hash = content_hash(&passage.text);
+        let transaction = self.connection.transaction()?;
+        for (i, (window, vector)) in windows.iter().zip(vectors).enumerate() {
+            ensure!(
+                vector.len() == self.dimensions && vector.iter().all(|v| v.is_finite()),
+                "Invalid persisted vector"
+            );
+            let norm = vector.iter().map(|value| value * value).sum::<f32>().sqrt();
+            ensure!(
+                norm.is_finite() && (norm - 1.0).abs() < 1e-3,
+                "Persisted embeddings must be L2 normalized"
+            );
+            ensure!(
+                passage.text.get(window.start_byte..window.end_byte) == Some(window.text.as_str()),
+                "Window does not match passage source range"
+            );
+            let bytes: Vec<u8> = vector
+                .iter()
+                .flat_map(|value| value.to_le_bytes())
+                .collect();
+            transaction.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector)
+                VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)", params![generation, passage.id,
+                (first_ordinal+i) as i64, passage.path, passage.source.kind, passage.source.number,
+                hash, window.start_byte as i64, window.end_byte as i64, window.text, bytes])?;
+        }
+        transaction.commit()?;
+        Ok(())
+    }
+
+    pub(crate) fn activate(&mut self, generation: i64) -> Result<()> {
+        // A failed graph build or serialization leaves the old generation active.
+        let ann = self.build_ann(generation)?;
+        let length = ann.serialized_length();
+        ensure!(
+            length <= 512 * 1024 * 1024,
+            "Serialized ANN exceeds 512 MiB generation budget"
+        );
+        let mut serialized = vec![0_u8; length];
+        ann.save_to_buffer(&mut serialized)?;
+        let checksum = format!("{:x}", Sha256::digest(&serialized));
+        let transaction = self.connection.transaction()?;
+        ensure!(
+            transaction.query_row(
+                "SELECT EXISTS(SELECT 1 FROM semantic_generations WHERE id=?1)",
+                [generation],
+                |row| row.get::<_, bool>(0)
+            )?,
+            "Missing staging generation"
+        );
+        transaction.execute(
+            "UPDATE semantic_generations SET ann_blob=?1,ann_sha256=?2 WHERE id=?3",
+            params![serialized, checksum, generation],
+        )?;
+        transaction.execute(
+            "UPDATE semantic_meta SET active_generation=?1 WHERE singleton=1",
+            [generation],
+        )?;
+        transaction.execute(
+            "DELETE FROM semantic_generations WHERE id!=?1",
+            [generation],
+        )?;
+        transaction.execute_batch("UPDATE semantic_meta SET dirty_floor=max(dirty_floor,corpus_revision-100000) WHERE singleton=1;
+            DELETE FROM semantic_dirty_paths WHERE revision<(SELECT dirty_floor FROM semantic_meta WHERE singleton=1);")?;
+        transaction.commit()?;
+        self.ann_cache = Some((generation, ann));
+        self.eligibility_cache = None;
+        Ok(())
+    }
+
+    fn new_ann(&self) -> Result<Index> {
+        Ok(Index::new(&IndexOptions {
+            dimensions: self.dimensions,
+            metric: MetricKind::Cos,
+            quantization: ScalarKind::F32,
+            connectivity: 16,
+            expansion_add: 128,
+            expansion_search: 64,
+            multi: false,
+        })?)
+    }
+
+    fn build_ann(&self, generation: i64) -> Result<Index> {
+        let count: usize = self.connection.query_row(
+            "SELECT count(*) FROM semantic_chunks WHERE generation=?1",
+            [generation],
+            |row| row.get(0),
+        )?;
+        ensure!(
+            count <= MAX_SCAN_CHUNKS,
+            "ANN generation exceeds {MAX_SCAN_CHUNKS} chunks"
+        );
+        let ann = self.new_ann()?;
+        ann.reserve_capacity_and_threads(count.max(1), 1)?;
+        let mut statement = self.connection.prepare(
+            "SELECT rowid,vector FROM semantic_chunks WHERE generation=?1 ORDER BY rowid",
+        )?;
+        let mut rows = statement.query([generation])?;
+        while let Some(row) = rows.next()? {
+            let key: i64 = row.get(0)?;
+            let bytes: Vec<u8> = row.get(1)?;
+            ensure!(
+                bytes.len() == self.dimensions * 4,
+                "Invalid staging embedding length"
+            );
+            let vector: Vec<f32> = bytes
+                .as_chunks::<4>()
+                .0
+                .iter()
+                .map(|bytes| f32::from_le_bytes(*bytes))
+                .collect();
+            ensure!(
+                vector.iter().all(|v| v.is_finite()),
+                "Invalid staging embedding values"
+            );
+            ann.add(u64::try_from(key)?, &vector)?;
+        }
+        Ok(ann)
+    }
+
+    pub(crate) fn abandon(&mut self, generation: i64) -> Result<()> {
+        self.connection.execute("DELETE FROM semantic_generations WHERE id=?1 AND id!=COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1)", [generation])?;
+        Ok(())
+    }
+
+    pub(crate) fn search(
+        &mut self,
+        query: &[f32],
+        fingerprint: &str,
+        limit: usize,
+        options: &SearchOptions,
+    ) -> Result<Vec<SearchResult>> {
+        ensure!(
+            query.len() == self.dimensions && query.iter().all(|v| v.is_finite()),
+            "Invalid query vector"
+        );
+        if limit == 0 {
+            return Ok(Vec::new());
+        }
+        // A consistent read snapshot covers eligibility and materialization.
+        let snapshot = self.connection.unchecked_transaction()?;
+        let active:Option<i64>=snapshot.query_row("SELECT g.id FROM semantic_generations g JOIN semantic_meta m ON m.active_generation=g.id WHERE g.fingerprint=?1 AND g.dimensions=?2",
+            params![fingerprint,self.dimensions as i64],|row|row.get(0)).optional()?;
+        let Some(active) = active else {
+            return Ok(Vec::new());
+        };
+        if !self
+            .ann_cache
+            .as_ref()
+            .is_some_and(|(generation, _)| *generation == active)
+        {
+            let length: Option<usize> = snapshot.query_row(
+                "SELECT length(ann_blob) FROM semantic_generations WHERE id=?1",
+                [active],
+                |row| row.get(0),
+            )?;
+            ensure!(
+                length.is_some_and(|bytes| bytes <= 512 * 1024 * 1024),
+                "Missing or oversized ANN generation; rebuild embeddings explicitly"
+            );
+            let (serialized, checksum): (Vec<u8>, String) = snapshot.query_row(
+                "SELECT ann_blob,ann_sha256 FROM semantic_generations WHERE id=?1",
+                [active],
+                |row| Ok((row.get(0)?, row.get(1)?)),
+            )?;
+            ensure!(
+                format!("{:x}", Sha256::digest(&serialized)) == checksum,
+                "ANN generation SHA256 mismatch; rebuild embeddings explicitly"
+            );
+            let ann = self.new_ann()?;
+            ann.load_from_buffer(&serialized)?;
+            ensure!(
+                ann.dimensions() == self.dimensions,
+                "ANN dimensions differ from embedding provider"
+            );
+            ann.reserve_capacity_and_threads(ann.size().max(1), 1)?;
+            self.ann_cache = Some((active, ann));
+        }
+        let folder = options
+            .folder
+            .as_ref()
+            .map(std::fs::canonicalize)
+            .transpose()?
+            .map(|p| {
+                format!(
+                    "{}{}",
+                    p.to_string_lossy().trim_end_matches(['/', '\\']),
+                    std::path::MAIN_SEPARATOR
+                )
+            });
+        let extension = options
+            .extension
+            .as_ref()
+            .map(|ext| ext.trim_start_matches('.').to_ascii_lowercase());
+        let min_size = options.min_size.map(i64::try_from).transpose()?;
+        let max_size = options.max_size.map(i64::try_from).transpose()?;
+        let filter = FilterKey {
+            folder,
+            extension,
+            min_size,
+            max_size,
+            modified_after: options.modified_after_unix_ms,
+            modified_before: options.modified_before_unix_ms,
+        };
+        let (revision, floor): (i64, i64) = snapshot.query_row(
+            "SELECT corpus_revision,dirty_floor FROM semantic_meta WHERE singleton=1",
+            [],
+            |row| Ok((row.get(0)?, row.get(1)?)),
+        )?;
+        let existing = self.eligibility_cache.take();
+        let mut cache = match existing {
+            Some(mut cache)
+                if cache.generation == active
+                    && cache.filter == filter
+                    && cache.revision >= floor =>
+            {
+                if cache.revision != revision {
+                    let mut statement=snapshot.prepare("SELECT path FROM semantic_dirty_paths WHERE revision>?1 AND revision<=?2 ORDER BY path LIMIT 1001")?;
+                    let paths = statement
+                        .query_map(params![cache.revision, revision], |row| {
+                            row.get::<_, String>(0)
+                        })?
+                        .collect::<rusqlite::Result<Vec<_>>>()?;
+                    if paths.len() > 1000 {
+                        cache = collect_eligibility(&snapshot, active, revision, &filter, None)?;
+                    } else {
+                        for path in paths {
+                            if let Some(old) = cache.by_path.remove(&path) {
+                                for key in old {
+                                    Arc::make_mut(&mut cache.keys).remove(&key);
+                                }
+                            }
+                            let changed = collect_eligibility(
+                                &snapshot,
+                                active,
+                                revision,
+                                &filter,
+                                Some(&path),
+                            )?;
+                            Arc::make_mut(&mut cache.keys).extend(changed.keys.iter().copied());
+                            cache.by_path.extend(changed.by_path);
+                        }
+                        cache.revision = revision;
+                    }
+                }
+                cache
+            }
+            _ => collect_eligibility(&snapshot, active, revision, &filter, None)?,
+        };
+        ensure!(
+            cache.keys.len() <= MAX_SCAN_CHUNKS,
+            "Eligibility cache exceeds chunk resource budget"
+        );
+        cache.revision = revision;
+        let eligible = Arc::clone(&cache.keys);
+        self.eligibility_cache = Some(cache);
+        if eligible.is_empty() {
+            return Ok(Vec::new());
+        }
+        let ann = &self.ann_cache.as_ref().expect("loaded ANN cache").1;
+        let mut candidate_count = limit.saturating_mul(4).min(eligible.len()).max(1);
+        let mut results;
+        loop {
+            let matches =
+                ann.filtered_search(query, candidate_count, |key| eligible.contains(&key))?;
+            let mut ranked: Vec<_> = matches.keys.into_iter().zip(matches.distances).collect();
+            ranked.sort_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0)));
+            let mut best = HashMap::<String, SearchResult>::new();
+            let mut materialize=snapshot.prepare("SELECT c.path,f.filename,c.snippet,c.kind,c.number FROM semantic_chunks c JOIN files f ON f.path=c.path WHERE c.rowid=?1 AND c.generation=?2")?;
+            for (key, distance) in ranked {
+                let cosine = 1.0 - distance;
+                ensure!(cosine.is_finite(), "ANN produced nonfinite distance");
+                if cosine < MIN_COSINE {
+                    continue;
+                }
+                let result =
+                    materialize.query_row(params![i64::try_from(key)?, active], |row| {
+                        Ok(SearchResult {
+                            path: row.get(0)?,
+                            filename: row.get(1)?,
+                            snippet: row.get(2)?,
+                            score: f64::from(cosine),
+                            match_kind: "semantic".into(),
+                            source: Some(SourceLocation {
+                                kind: row.get(3)?,
+                                number: row.get(4)?,
+                            }),
+                            extraction_error: None,
+                        })
+                    })?;
+                if best.contains_key(&result.path) {
+                    continue;
+                }
+                best.insert(result.path.clone(), result);
+            }
+            results = best.into_values().collect::<Vec<_>>();
+            if results.len() >= limit || candidate_count == eligible.len() {
+                break;
+            }
+            candidate_count = candidate_count.saturating_mul(2).min(eligible.len());
+        }
+        results.sort_by(|a, b| {
+            b.score
+                .total_cmp(&a.score)
+                .then_with(|| a.path.cmp(&b.path))
+        });
+        results.truncate(limit);
+        snapshot.commit()?;
+        Ok(results)
+    }
+
+    #[cfg(test)]
+    fn exact_search(
+        &self,
+        query: &[f32],
+        fingerprint: &str,
+        limit: usize,
+        options: &SearchOptions,
+    ) -> Result<Vec<SearchResult>> {
+        ensure!(
+            query.len() == self.dimensions && query.iter().all(|v| v.is_finite()),
+            "Invalid query vector"
+        );
+        let active: Option<i64> = self.connection.query_row("SELECT g.id FROM semantic_generations g JOIN semantic_meta m ON m.active_generation=g.id WHERE g.fingerprint=?1 AND g.dimensions=?2", params![fingerprint,self.dimensions as i64], |row| row.get(0)).optional()?;
+        let Some(active) = active else {
+            return Ok(Vec::new());
+        };
+        let folder = options
+            .folder
+            .as_ref()
+            .map(std::fs::canonicalize)
+            .transpose()?
+            .map(|p| {
+                format!(
+                    "{}{}",
+                    p.to_string_lossy().trim_end_matches(['/', '\\']),
+                    std::path::MAIN_SEPARATOR
+                )
+            });
+        let extension = options
+            .extension
+            .as_ref()
+            .map(|ext| ext.trim_start_matches('.').to_ascii_lowercase());
+        let min_size = options.min_size.map(i64::try_from).transpose()?;
+        let max_size = options.max_size.map(i64::try_from).transpose()?;
+        let mut statement = self.connection.prepare("SELECT c.passage_id,c.path,f.filename,c.snippet,c.vector,c.content_hash,p.text,c.kind,c.number
+            FROM semantic_chunks c JOIN passages p ON p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
+            JOIN files f ON f.path=c.path WHERE c.generation=?1
+            AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2)
+            AND (?3 IS NULL OR f.extension=?3)
+            AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
+            AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
+            ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
+        // Fetch one extra to detect the baseline cap instead of silently hiding
+        // candidates. Metadata filtering precedes this cap and all top-k limits.
+        let mut rows = statement.query(params![
+            active,
+            folder,
+            extension,
+            min_size,
+            max_size,
+            options.modified_after_unix_ms,
+            options.modified_before_unix_ms,
+            MAX_SCAN_CHUNKS as i64 + 1
+        ])?;
+        let mut hashes = HashMap::<i64, String>::new();
+        let mut best = HashMap::<String, SearchResult>::new();
+        let mut scanned = 0;
+        while let Some(row) = rows.next()? {
+            scanned += 1;
+            ensure!(
+                scanned <= MAX_SCAN_CHUNKS,
+                "Exact-vector baseline exceeds {MAX_SCAN_CHUNKS} chunks; ANN integration required"
+            );
+            let passage_id: i64 = row.get(0)?;
+            let current_hash = if let Some(hash) = hashes.get(&passage_id) {
+                hash.clone()
+            } else {
+                let hash = content_hash(&row.get::<_, String>(6)?);
+                hashes.insert(passage_id, hash.clone());
+                hash
+            };
+            if current_hash != row.get::<_, String>(5)? {
+                continue;
+            }
+            let bytes: Vec<u8> = row.get(4)?;
+            ensure!(
+                bytes.len() == self.dimensions * 4,
+                "Corrupt stored embedding length"
+            );
+            let mut cosine = 0.0;
+            for (dimension, value) in bytes.as_chunks::<4>().0.iter().enumerate() {
+                let value = f32::from_le_bytes(*value);
+                ensure!(value.is_finite(), "Corrupt stored embedding value");
+                cosine += value * query[dimension];
+            }
+            if cosine < MIN_COSINE {
+                continue;
+            }
+            let path: String = row.get(1)?;
+            if best
+                .get(&path)
+                .is_some_and(|result| result.score >= f64::from(cosine))
+            {
+                continue;
+            }
+            best.insert(
+                path.clone(),
+                SearchResult {
+                    path,
+                    filename: row.get(2)?,
+                    snippet: row.get(3)?,
+                    score: f64::from(cosine),
+                    match_kind: "semantic".into(),
+                    source: Some(SourceLocation {
+                        kind: row.get(7)?,
+                        number: row.get(8)?,
+                    }),
+                    extraction_error: None,
+                },
+            );
+        }
+        let mut results: Vec<_> = best.into_values().collect();
+        results.sort_by(|a, b| {
+            b.score
+                .total_cmp(&a.score)
+                .then_with(|| a.path.cmp(&b.path))
+        });
+        results.truncate(limit);
+        Ok(results)
+    }
+}
+
+fn collect_eligibility(
+    connection: &Connection,
+    generation: i64,
+    revision: i64,
+    filter: &FilterKey,
+    path: Option<&str>,
+) -> Result<EligibilityCache> {
+    let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path FROM semantic_chunks c
+        JOIN passages p ON p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
+        JOIN files f ON f.path=c.path WHERE c.generation=?1
+        AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
+        AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
+        AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
+        AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
+    let mut rows = statement.query(params![
+        generation,
+        filter.folder,
+        filter.extension,
+        filter.min_size,
+        filter.max_size,
+        filter.modified_after,
+        filter.modified_before,
+        MAX_SCAN_CHUNKS as i64 + 1,
+        path
+    ])?;
+    let mut hashes = HashMap::<i64, String>::new();
+    let mut keys = HashSet::new();
+    let mut by_path = HashMap::<String, Vec<u64>>::new();
+    let mut count = 0;
+    while let Some(row) = rows.next()? {
+        count += 1;
+        ensure!(
+            count <= MAX_SCAN_CHUNKS,
+            "Corpus exceeds {MAX_SCAN_CHUNKS} chunk resource budget"
+        );
+        let passage_id: i64 = row.get(1)?;
+        let hash = if let Some(hash) = hashes.get(&passage_id) {
+            hash.clone()
+        } else {
+            let hash = content_hash(&row.get::<_, String>(3)?);
+            hashes.insert(passage_id, hash.clone());
+            hash
+        };
+        if hash == row.get::<_, String>(2)? {
+            let key = u64::try_from(row.get::<_, i64>(0)?)?;
+            keys.insert(key);
+            by_path.entry(row.get(4)?).or_default().push(key);
+        }
+    }
+    Ok(EligibilityCache {
+        generation,
+        revision,
+        filter: filter.clone(),
+        keys: Arc::new(keys),
+        by_path,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use searchi_core::SearchIndex;
+    const DIMENSIONS: usize = 768;
+
+    #[test]
+    fn generation_persists_failed_staging_preserves_active_and_changes_hide_stale_vectors(
+    ) -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let root = directory.path().join("docs");
+        std::fs::create_dir(&root)?;
+        let db = directory.path().join("index.db");
+        std::fs::write(root.join("notes.txt"), "original relevant content")?;
+        let mut index = SearchIndex::open(&db)?;
+        index.index_folder(&root)?;
+        let passage = index.passage_batch(0, 1)?.remove(0);
+        let mut vector = vec![0.0; DIMENSIONS];
+        vector[0] = 1.0;
+        let windows = [TextWindow {
+            text: passage.text.clone(),
+            start_byte: 0,
+            end_byte: passage.text.len(),
+        }];
+        let mut store = Store::open(&db, DIMENSIONS)?;
+        let generation = store.begin("test-storage-vectors")?;
+        store.insert(generation, &passage, 0, &windows, &[vector.clone()])?;
+        store.activate(generation)?;
+        assert_eq!(
+            store.exact_search(
+                &vector,
+                "test-storage-vectors",
+                10,
+                &SearchOptions::default()
+            )?[0]
+                .path,
+            store.search(
+                &vector,
+                "test-storage-vectors",
+                10,
+                &SearchOptions::default()
+            )?[0]
+                .path
+        );
+        drop(store);
+        let mut store = Store::open(&db, DIMENSIONS)?;
+        assert_eq!(
+            store
+                .search(
+                    &vector,
+                    "test-storage-vectors",
+                    10,
+                    &SearchOptions::default()
+                )?
+                .len(),
+            1
+        );
+        assert!(store
+            .search(&vector, "different-model", 10, &SearchOptions::default())?
+            .is_empty());
+        let staging = store.begin("new-model")?;
+        assert_eq!(
+            store
+                .search(
+                    &vector,
+                    "test-storage-vectors",
+                    10,
+                    &SearchOptions::default()
+                )?
+                .len(),
+            1
+        );
+        store.abandon(staging)?;
+        std::fs::write(
+            root.join("notes.txt"),
+            "replacement unrelated content longer than before",
+        )?;
+        index.index_folder(&root)?;
+        assert!(store
+            .search(
+                &vector,
+                "test-storage-vectors",
+                10,
+                &SearchOptions::default()
+            )?
+            .is_empty());
+        std::fs::remove_file(root.join("notes.txt"))?;
+        index.index_folder(&root)?;
+        assert!(store
+            .search(
+                &vector,
+                "test-storage-vectors",
+                10,
+                &SearchOptions::default()
+            )?
+            .is_empty());
+        Ok(())
+    }
+
+    #[test]
+    fn ann_filters_before_top_k_checks_checksum_and_status_tracks_corpus_changes() -> Result<()> {
+        let directory = tempfile::tempdir()?;
+        let root = directory.path().join("docs");
+        std::fs::create_dir(&root)?;
+        let db = directory.path().join("index.db");
+        std::fs::write(root.join("closest.md"), "closest metadata excluded source")?;
+        std::fs::write(root.join("allowed.txt"), "allowed metadata source")?;
+        let mut lexical = SearchIndex::open(&db)?;
+        lexical.index_folder(&root)?;
+        let passages = lexical.passage_batch(0, 10)?;
+        let mut store = Store::open(&db, DIMENSIONS)?;
+        let fingerprint = crate::generation_fingerprint(searchi_semantic::GEMMA_FINGERPRINT);
+        let generation = store.begin(&fingerprint)?;
+        let mut query = vec![0.0; DIMENSIONS];
+        query[0] = 1.0;
+        for passage in &passages {
+            let mut vector = query.clone();
+            if passage.filename == "allowed.txt" {
+                vector[0] = 0.8;
+                vector[1] = 0.6;
+            }
+            let windows = [TextWindow {
+                text: passage.text.clone(),
+                start_byte: 0,
+                end_byte: passage.text.len(),
+            }];
+            store.insert(generation, passage, 0, &windows, &[vector])?;
+        }
+        store.activate(generation)?;
+        let options = SearchOptions {
+            extension: Some("txt".into()),
+            ..Default::default()
+        };
+        assert_eq!(
+            store.search(&query, &fingerprint, 1, &options)?[0].filename,
+            "allowed.txt"
+        );
+        let warmed_hashes = HASH_CALLS.with(std::cell::Cell::get);
+        let _ = store.search(&query, &fingerprint, 1, &options)?;
+        assert_eq!(
+            HASH_CALLS.with(std::cell::Cell::get),
+            warmed_hashes,
+            "Warm query rehashed unchanged corpus"
+        );
+        assert_eq!(
+            store.exact_search(&query, &fingerprint, 1, &options)?[0].filename,
+            "allowed.txt"
+        );
+        assert!(!crate::HybridSearch::embedding_status(&db)?.stale);
+        std::fs::write(
+            root.join("allowed.txt"),
+            "replacement changes the generation revision",
+        )?;
+        lexical.index_folder(&root)?;
+        let status = crate::HybridSearch::embedding_status(&db)?;
+        let before_delta = HASH_CALLS.with(std::cell::Cell::get);
+        assert!(store.search(&query, &fingerprint, 1, &options)?.is_empty());
+        assert!(
+            HASH_CALLS.with(std::cell::Cell::get) - before_delta <= 1,
+            "Single file change rehashed unrelated passages"
+        );
+        assert!(status.stale);
+        assert_eq!(status.pending, 2);
+        store.connection.execute(
+            "UPDATE semantic_generations SET ann_blob=x'010203' WHERE id=?1",
+            [generation],
+        )?;
+        drop(store);
+        let mut store = Store::open(&db, DIMENSIONS)?;
+        assert!(store
+            .search(&query, &fingerprint, 1, &SearchOptions::default())
+            .unwrap_err()
+            .to_string()
+            .contains("SHA256 mismatch"));
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+#[path = "ann_benchmark.rs"]
+mod ann_benchmark;