feat: integrate Google semantic search and branded desktop launcher

crates/searchi-core/src/lib.rs

@@ -16,7 +16,7 @@ use std::{
 };
 use walkdir::WalkDir;
 
-const SCHEMA_VERSION: i64 = 7;
+const SCHEMA_VERSION: i64 = 9;
 const MAX_RESULTS: usize = 1000;
 
 #[derive(Debug, Default, Serialize)]
@@ -214,7 +214,18 @@ impl SearchIndex {
             }
             initialization.execute_batch(
                 "CREATE VIRTUAL TABLE IF NOT EXISTS names_vocab USING fts5vocab(names,'row');
-                PRAGMA user_version=7;",
+                CREATE TABLE IF NOT EXISTS name_frequencies(term TEXT PRIMARY KEY, doc INTEGER NOT NULL);
+                CREATE INDEX IF NOT EXISTS name_frequencies_popularity ON name_frequencies(doc DESC);
+                CREATE TABLE IF NOT EXISTS name_statistics_state(singleton INTEGER PRIMARY KEY CHECK(singleton=1), dirty INTEGER NOT NULL);
+                INSERT OR IGNORE INTO name_statistics_state VALUES(1,1);
+                CREATE TRIGGER IF NOT EXISTS files_statistics_insert AFTER INSERT ON files BEGIN
+                    UPDATE name_statistics_state SET dirty=1 WHERE singleton=1; END;
+                CREATE TRIGGER IF NOT EXISTS files_statistics_delete AFTER DELETE ON files BEGIN
+                    UPDATE name_statistics_state SET dirty=1 WHERE singleton=1; END;
+                CREATE TRIGGER IF NOT EXISTS files_statistics_update AFTER UPDATE OF filename,path ON files
+                    WHEN new.filename!=old.filename OR new.path!=old.path BEGIN
+                    UPDATE name_statistics_state SET dirty=1 WHERE singleton=1; END;
+                PRAGMA user_version=9;",
             )?;
             initialization.commit()?;
         }
@@ -225,6 +236,89 @@ impl SearchIndex {
         })
     }
 
+    /// Refresh candidate-frequency statistics off the foreground search path.
+    /// Stale/missing estimates can affect speed, never literal-match correctness.
+    /// The app invokes this during background startup and after reconciliation.
+    pub fn prewarm(&mut self) -> Result<usize> {
+        let dirty: bool = self.connection.query_row(
+            "SELECT dirty FROM name_statistics_state WHERE singleton=1",
+            [],
+            |row| row.get(0),
+        )?;
+        if dirty {
+            let transaction = self
+                .connection
+                .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+            transaction.execute("DELETE FROM name_frequencies", [])?;
+            transaction.execute(
+                "INSERT INTO name_frequencies(term,doc) SELECT term,doc FROM names_vocab",
+                [],
+            )?;
+            transaction.execute(
+                "UPDATE name_statistics_state SET dirty=0 WHERE singleton=1",
+                [],
+            )?;
+            transaction.commit()?;
+        }
+        let mut statement = self
+            .connection
+            .prepare("SELECT term,doc FROM name_frequencies ORDER BY doc DESC LIMIT 4096")?;
+        let rows = statement.query_map([], |row| {
+            Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
+        })?;
+        let mut frequencies = self.gram_frequencies.borrow_mut();
+        frequencies.clear();
+        for row in rows {
+            let (term, frequency) = row?;
+            frequencies.insert(term, frequency);
+        }
+        let cached = frequencies.len();
+        drop(frequencies);
+        // Exercise real query paths using representative corpus data, rather
+        // than paying postings, snippet and SQLite page initialization on input.
+        let mut representative = self.connection.prepare(
+            "SELECT filename FROM files WHERE id=(SELECT min(id) FROM files)
+             OR id=(SELECT max(id) FROM files)",
+        )?;
+        let names = representative
+            .query_map([], |row| row.get::<_, String>(0))?
+            .collect::<rusqlite::Result<Vec<_>>>()?;
+        for name in &names {
+            self.search(name, 20)?;
+        }
+        // Full names often anchor at sparse identifiers. Also exercise actual
+        // alphabetic name tokens to initialize common prefix/type postings.
+        let words = names
+            .iter()
+            .flat_map(|name| {
+                name.split(|character: char| !character.is_alphabetic())
+                    .filter(|word| word.chars().count() >= 3)
+                    .take(3)
+            })
+            .collect::<BTreeSet<_>>();
+        for word in words {
+            self.search(word, 20)?;
+        }
+        let text: Option<String> = self
+            .connection
+            .query_row(
+                "SELECT substr(content,1,256) FROM files WHERE content!='' ORDER BY id LIMIT 1",
+                [],
+                |row| row.get(0),
+            )
+            .optional()?;
+        if let Some(text) = text {
+            for term in text
+                .split_whitespace()
+                .filter(|term| term.chars().all(char::is_alphanumeric))
+                .take(3)
+            {
+                self.search(term, 20)?;
+            }
+        }
+        Ok(cached)
+    }
+
     /// Reconcile one explicitly selected root. Symlinks are never followed.
     /// Every file receives a filename record, including unsupported formats.
     pub fn index_folder(&mut self, root: impl AsRef<Path>) -> Result<IndexReport> {
@@ -441,16 +535,17 @@ impl SearchIndex {
                 .modified_before_unix_ms
                 .map_or(Value::Null, Value::Integer),
         ];
+        let has_filters = filter_values.iter().any(|value| *value != Value::Null);
         let mut results = Vec::new();
         let mut selected = Vec::<i64>::new();
         // Each lane filters its entire candidate set before LIMIT. There is no
         // hidden pre-filter candidate cap that can hide later matching files.
         let trigram = query.chars().count() >= 3;
         // A long FTS phrase may walk common leading trigram postings even when
-        // later trigrams are rare. Use the rarest known trigram as a candidate
-        // superset, then verify the whole literal substring. The 256-term lookup
-        // budget can only enlarge candidates; it never caps results. Frequency
-        // lookups are cached because FTS vocabulary counts have a first-use cost.
+        // later trigrams are rare. Build a frequency-bounded positional anchor,
+        // then verify the whole literal substring. The 256-term lookup budget
+        // can only enlarge candidates; it never caps results. Frequencies come
+        // from persisted background statistics and a bounded reader cache.
         let characters = query.chars().collect::<Vec<_>>();
         let grams = characters
             .windows(3)
@@ -461,7 +556,7 @@ impl SearchIndex {
             .collect::<Vec<_>>();
         let mut vocabulary = self
             .connection
-            .prepare("SELECT doc FROM names_vocab WHERE term=?1")?;
+            .prepare("SELECT doc FROM name_frequencies WHERE term=?1")?;
         let mut cache = self.gram_frequencies.borrow_mut();
         if cache.len() + grams.len() > 4096 {
             cache.clear();
@@ -485,12 +580,8 @@ impl SearchIndex {
             .collect::<Result<Vec<_>>>()?;
         rare_first.sort();
         drop(cache);
-        let grams = rare_first
-            .into_iter()
-            .take(1)
-            .map(|(_, gram)| format!("\"{}\"", gram.replace('"', "\"\"")))
-            .collect::<Vec<_>>()
-            .join(" AND ");
+        let anchor = substring_anchor(&characters, &rare_first);
+        let grams = format!("\"{}\"", anchor.replace('"', "\"\""));
         let mut lanes = vec![
             (
                 "exact_filename",
@@ -585,10 +676,25 @@ impl SearchIndex {
             } else {
                 "''"
             };
-            let sql = format!(
-                "SELECT f.id,f.path,f.filename,{snippet},{rank},f.extraction_error FROM {from}
+            let sql = if kind == "content" && !has_filters {
+                // Rank compact row IDs before materializing paths and snippets.
+                // Filtered requests retain the complete filter-before-LIMIT lane
+                // below; this fast path is only for genuinely unfiltered search.
+                let excluded = exclusions.replace("f.id", "rowid");
+                format!("WITH hits AS MATERIALIZED (
+                    SELECT rowid AS id,rank AS relevance FROM content_search
+                    WHERE content_search MATCH ?1 {excluded}
+                    ORDER BY rank,rowid LIMIT ?9)
+                    SELECT f.id,f.path,f.filename,snippet(content_search,0,'','',' … ',24),hits.relevance,f.extraction_error
+                    FROM hits CROSS JOIN files f ON f.id=hits.id
+                    CROSS JOIN content_search ON content_search.rowid=hits.id
+                    WHERE content_search MATCH ?1 ORDER BY hits.relevance,f.id")
+            } else {
+                format!(
+                    "SELECT f.id,f.path,f.filename,{snippet},{rank},f.extraction_error FROM {from}
                 WHERE {condition} {filters} {exclusions} ORDER BY {ordering} LIMIT ?9"
-            );
+                )
+            };
             let mut values = vec![Value::Text(expression), Value::Text(query.into())];
             values.extend(filter_values.iter().cloned());
             values.push(Value::Integer((limit - results.len()) as i64));
@@ -702,6 +808,52 @@ impl SearchIndex {
     }
 }
 
+/// Build a bounded positional anchor around the rarest trigram. Extending a
+/// phrase through common suffixes can force FTS to decode dense postings. Only
+/// adjacent grams within a small frequency factor are admitted here. Every
+/// literal match contains this anchor; SQL still verifies the entire substring.
+fn substring_anchor(characters: &[char], frequencies: &[(i64, String)]) -> String {
+    let Some((minimum, gram)) = frequencies.first() else {
+        return String::new();
+    };
+    let center = characters
+        .windows(3)
+        .position(|window| window.iter().collect::<String>() == *gram)
+        .unwrap_or(0);
+    let mut start = center;
+    let mut end = center + 3;
+    if *minimum > 0 {
+        let maximum = minimum.saturating_mul(4).max(32);
+        let lookup = frequencies
+            .iter()
+            .map(|(frequency, gram)| (gram.as_str(), *frequency))
+            .collect::<HashMap<_, _>>();
+        let eligible = |index: usize| {
+            lookup
+                .get(
+                    characters[index..index + 3]
+                        .iter()
+                        .collect::<String>()
+                        .as_str(),
+                )
+                .is_some_and(|frequency| *frequency <= maximum)
+        };
+        loop {
+            let previous = (start, end);
+            if start > 0 && end - start < 32 && eligible(start - 1) {
+                start -= 1;
+            }
+            if end < characters.len() && end - start < 32 && eligible(end - 2) {
+                end += 1;
+            }
+            if previous == (start, end) {
+                break;
+            }
+        }
+    }
+    characters[start..end].iter().collect()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;