feat: add multimodal search and granular performance diagnostics

crates/searchi-service/src/lib.rs

@@ -1,10 +1,15 @@
 //! Shared offline hybrid search using real Google embeddings and persisted,
 //! filtered HNSW retrieval. Production-scale validation remains a release gate.
+mod media;
+mod routing;
+pub use routing::SemanticOrigin;
 mod store;
+pub use searchi_media::{FfmpegTools, MediaLimits};
 
 use anyhow::{ensure, Result};
+use searchi_core::metrics::{MetricsMode, SpanToken, TraceCollector};
 use searchi_core::SearchIndex;
-pub use searchi_core::{SearchOptions, SearchResult};
+pub use searchi_core::{SearchMetrics, SearchOptions, SearchResponse, SearchResult};
 pub use searchi_semantic::WorkerInfo;
 use searchi_semantic::{GemmaEmbedder, TextWindow, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
 use serde::Serialize;
@@ -17,7 +22,24 @@ const BYTE_OVERLAP: usize = 512;
 const MAX_PASSAGE_WINDOWS: usize = 2048;
 const CHUNK_POLICY: &str = "byte64k-overlap512-title512-v1";
 fn generation_fingerprint(model: &str) -> String {
-    format!("{model}:{CHUNK_POLICY}")
+    format!(
+        "{model}:{CHUNK_POLICY}:{}:image280-frame70-audio30s-overlap2-distributed240-scene120-v1",
+        searchi_semantic::GEMMA_MEDIA_FINGERPRINT
+    )
+}
+
+fn reusable_policy(expected: &str, active: &str) -> bool {
+    if active == expected || active == format!("{expected}:text-only") {
+        return true;
+    }
+    let Some(policy) = active.strip_prefix(&format!("{expected}:custom-")) else {
+        return false;
+    };
+    let values = policy
+        .split('-')
+        .map(str::parse::<u64>)
+        .collect::<std::result::Result<Vec<_>, _>>();
+    matches!(values,Ok(v) if v.len()==8 && v[0]>=2 && (5000..=30000).contains(&v[3]) && v[4]<v[3] && v[5]>0 && v[6]<=u32::MAX as u64)
 }
 
 /// Model-specific query/document prompts and actual tokenizer windows belong
@@ -28,6 +50,18 @@ pub trait EmbeddingProvider: Send {
     fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>>;
     fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result<Vec<Vec<f32>>>;
     fn embed_query(&mut self, query: &str) -> Result<Vec<f32>>;
+    fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
+        self.embed_query(query)
+    }
+    fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
+        Default::default()
+    }
+    fn embed_images(&mut self, _paths: &[&Path], _budget: u32) -> Result<Vec<Vec<f32>>> {
+        anyhow::bail!("Provider does not support real images")
+    }
+    fn embed_audio(&mut self, _pcm: &[f32]) -> Result<Vec<f32>> {
+        anyhow::bail!("Provider does not support real audio")
+    }
     fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
         None
     }
@@ -49,13 +83,34 @@ impl EmbeddingProvider for GemmaEmbedder {
     fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
         self.embed_query(query)
     }
+    fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
+        GemmaEmbedder::embed_query_debug(self, query)
+    }
+    fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
+        GemmaEmbedder::query_diagnostics(self)
+    }
+    fn embed_images(&mut self, paths: &[&Path], budget: u32) -> Result<Vec<Vec<f32>>> {
+        self.embed_images_with_budget(paths, budget)
+    }
+    fn embed_audio(&mut self, pcm: &[f32]) -> Result<Vec<f32>> {
+        self.embed_audio_pcm(pcm)
+    }
     fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
         Some(self.info.clone())
     }
 }
 
+#[derive(Debug, Serialize)]
+pub struct MediaFailure {
+    pub path: String,
+    pub message: String,
+}
+
 #[derive(Debug, Serialize)]
 pub struct EmbeddingReport {
+    pub media_files: usize,
+    pub media_failures: usize,
+    pub failures: Vec<MediaFailure>,
     pub passages: usize,
     pub chunks: usize,
     pub generation: i64,
@@ -64,6 +119,7 @@ pub struct EmbeddingReport {
 
 #[derive(Debug, Serialize)]
 pub struct EmbeddingStatus {
+    pub failures: Vec<MediaFailure>,
     pub generation: Option<i64>,
     pub chunks: usize,
     pub model_fingerprint: Option<String>,
@@ -98,9 +154,10 @@ impl HybridSearch {
             |row| row.get(0),
         )?;
         let pending: usize =
-            connection.query_row("SELECT count(*) FROM passages", [], |row| row.get(0))?;
+            connection.query_row("SELECT (SELECT count(*) FROM passages)+(SELECT count(*) FROM files WHERE extension IN ('png','jpg','jpeg','webp','bmp','mp4','mkv','mov','avi','webm','m4v','mpeg','mpg','mp3','flac','wav','m4a','ogg','opus','aac','aiff','aif'))", [], |row| row.get(0))?;
         if !exists {
             return Ok(EmbeddingStatus {
+                failures: Vec::new(),
                 generation: None,
                 chunks: 0,
                 model_fingerprint: None,
@@ -113,8 +170,9 @@ impl HybridSearch {
             [],
             |row| row.get(0),
         )?;
-        if version < 5 {
+        if version < 6 {
             return Ok(EmbeddingStatus {
+                failures: Vec::new(),
                 generation: None,
                 chunks: 0,
                 model_fingerprint: None,
@@ -123,7 +181,7 @@ impl HybridSearch {
             });
         }
         ensure!(
-            version == 5,
+            version == 6,
             "Unsupported semantic schema version {version}"
         );
         let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
@@ -134,9 +192,19 @@ impl HybridSearch {
             [generation],
             |row| row.get(0),
         )?;
+        let mut failures_query=connection.prepare("SELECT path,message FROM semantic_media_errors WHERE generation=?1 ORDER BY path LIMIT 1000")?;
+        let failures = failures_query
+            .query_map([generation], |r| {
+                Ok(MediaFailure {
+                    path: r.get(0)?,
+                    message: r.get(1)?,
+                })
+            })?
+            .collect::<rusqlite::Result<Vec<_>>>()?;
         let stale = revision != Some(current)
             || fingerprint.as_deref() != Some(generation_fingerprint(GEMMA_FINGERPRINT).as_str());
         Ok(EmbeddingStatus {
+            failures,
             generation,
             chunks,
             model_fingerprint: fingerprint,
@@ -162,7 +230,12 @@ impl HybridSearch {
     ) -> Result<Self> {
         let lexical = SearchIndex::open(&db)?;
         let store = Store::open(db.as_ref(), embedder.dimensions())?;
-        let generation_fingerprint = generation_fingerprint(embedder.fingerprint());
+        let expected = generation_fingerprint(embedder.fingerprint());
+        let active = store.active_fingerprint()?;
+        let generation_fingerprint = match active {
+            Some(active) if reusable_policy(&expected, &active) => active,
+            _ => expected,
+        };
         Ok(Self {
             lexical,
             embedder,
@@ -189,6 +262,11 @@ impl HybridSearch {
     /// storage errors leave the previous generation active. A crashed build is
     /// reclaimed on the next explicit rebuild under an OS advisory lock.
     pub fn index_embeddings(&mut self) -> Result<EmbeddingReport> {
+        let previous_fingerprint = self.generation_fingerprint.clone();
+        self.generation_fingerprint = format!(
+            "{}:text-only",
+            generation_fingerprint(self.embedder.fingerprint())
+        );
         let _lock = self.store.build_lock()?;
         let generation = self.store.begin(&self.generation_fingerprint)?;
         let built = self.build(generation);
@@ -199,6 +277,7 @@ impl HybridSearch {
             }
             Err(error) => {
                 self.store.abandon(generation)?;
+                self.generation_fingerprint = previous_fingerprint;
                 Err(error)
             }
         }
@@ -206,6 +285,9 @@ impl HybridSearch {
 
     fn build(&mut self, generation: i64) -> Result<EmbeddingReport> {
         let mut report = EmbeddingReport {
+            media_files: 0,
+            media_failures: 0,
+            failures: Vec::new(),
             passages: 0,
             chunks: 0,
             generation,
@@ -253,21 +335,81 @@ impl HybridSearch {
         query: &str,
         limit: usize,
         options: &SearchOptions,
+    ) -> Result<Vec<SearchResult>> {
+        let mut trace = TraceCollector::new(MetricsMode::Basic);
+        self.search_traced(query, limit, options, &mut trace, None)
+    }
+    pub fn search_measured(
+        &mut self,
+        query: &str,
+        limit: usize,
+        options: &SearchOptions,
+        debug: bool,
+    ) -> Result<SearchResponse<SearchResult>> {
+        let mut trace = TraceCollector::new(if debug {
+            MetricsMode::Debug
+        } else {
+            MetricsMode::Basic
+        });
+        let results = self.search_traced(query, limit, options, &mut trace, None)?;
+        let metrics = trace.finish(results.len());
+        Ok(SearchResponse { results, metrics })
+    }
+    pub fn search_traced(
+        &mut self,
+        query: &str,
+        limit: usize,
+        options: &SearchOptions,
+        trace: &mut TraceCollector,
+        parent: Option<SpanToken>,
+    ) -> Result<Vec<SearchResult>> {
+        trace.timed("hybrid.search", parent, |trace, parent| {
+            self.search_inner(query, limit, options, trace, parent)
+        })
+    }
+    fn search_inner(
+        &mut self,
+        query: &str,
+        limit: usize,
+        options: &SearchOptions,
+        trace: &mut TraceCollector,
+        parent: Option<SpanToken>,
     ) -> Result<Vec<SearchResult>> {
         let candidates = limit.saturating_mul(4).clamp(20, 1000);
         // Core validates filters and query budgets even for empty requests.
         let mut lexical = self
             .lexical
-            .search_with_options(query, candidates, options)?;
+            .search_with_options_traced(query, candidates, options, trace, parent)?;
         if query.trim().is_empty() || limit == 0 {
             return Ok(Vec::new());
         }
-        promote_identifiers(query.trim(), &mut lexical);
-        let vector = self.embedder.embed_query(query.trim())?;
+        trace.timed("hybrid.identifier_priority", parent, |_, _| {
+            promote_identifiers(query.trim(), &mut lexical)
+        });
+        let embedding_span = trace.begin("semantic.query_embedding", parent);
+        let vector = if trace.is_debug() {
+            self.embedder.embed_query_debug(query.trim())
+        } else {
+            self.embedder.embed_query(query.trim())
+        };
+        trace.counter(embedding_span, "failed", u64::from(vector.is_err()));
+        trace.end(
+            embedding_span,
+            if trace.is_debug() {
+                self.embedder.query_diagnostics()
+            } else {
+                Default::default()
+            },
+        );
+        let vector = vector?;
         let semantic =
-            self.store
-                .search(&vector, &self.generation_fingerprint, candidates, options)?;
-        Ok(fuse(lexical, semantic, limit.min(1000)))
+            self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?;
+        let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| {
+            trace.counter(span, "lexical_files", lexical.len() as u64);
+            trace.counter(span, "semantic_files", semantic.len() as u64);
+            fuse(lexical, semantic, limit.min(1000))
+        });
+        Ok(results)
     }
 
     /// Semantic lane for reproducible relevance evaluation and diagnostics.
@@ -282,12 +424,7 @@ impl HybridSearch {
             return Ok(Vec::new());
         }
         let vector = self.embedder.embed_query(query.trim())?;
-        self.store.search(
-            &vector,
-            &self.generation_fingerprint,
-            limit.min(1000),
-            options,
-        )
+        self.routed_semantic(query, &vector, limit.min(1000), options)
     }
 }
 
@@ -417,6 +554,58 @@ fn fuse(
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[test]
+    fn failed_query_inference_retains_diagnostics_in_shared_trace() -> Result<()> {
+        // Failure-only provider: no fabricated embeddings are generated.
+        struct FailingProvider;
+        impl EmbeddingProvider for FailingProvider {
+            fn fingerprint(&self) -> &str {
+                "failure-injection"
+            }
+            fn dimensions(&self) -> usize {
+                768
+            }
+            fn split_windows(&mut self, _: &str, _: &str) -> Result<Vec<TextWindow>> {
+                anyhow::bail!("unused")
+            }
+            fn embed_documents(&mut self, _: &[&str], _: &str) -> Result<Vec<Vec<f32>>> {
+                anyhow::bail!("unused")
+            }
+            fn embed_query(&mut self, _: &str) -> Result<Vec<f32>> {
+                anyhow::bail!("injected worker failure")
+            }
+            fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
+                std::collections::BTreeMap::from([("request_write_us".into(), 7)])
+            }
+        }
+        let directory = tempfile::tempdir()?;
+        let db = directory.path().join("index.db");
+        let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
+        let mut trace = TraceCollector::new(MetricsMode::Debug);
+        assert!(service
+            .search_traced(
+                "ordinary query",
+                5,
+                &SearchOptions::default(),
+                &mut trace,
+                None
+            )
+            .is_err());
+        let metrics = trace.finish(0);
+        let spans = metrics.trace.unwrap();
+        let embedding = spans
+            .iter()
+            .find(|s| s.name == "semantic.query_embedding")
+            .unwrap();
+        assert_eq!(embedding.counters["failed"], 1);
+        assert_eq!(embedding.counters["request_write_us"], 7);
+        let parent = &spans[embedding.parent.unwrap()];
+        assert!(
+            embedding.start_ms + embedding.duration_ms
+                <= parent.start_ms + parent.duration_ms + 1e-6
+        );
+        Ok(())
+    }
     fn result(path: &str, kind: &str) -> SearchResult {
         SearchResult {
             path: path.into(),