feat: add multimodal search and granular performance diagnostics

apps/desktop/src-tauri/src/semantic_commands.rs

@@ -1,5 +1,6 @@
 //! Desktop-owned orchestration: managed model setup and persistent hybrid queries.
-use crate::{model_download, AppState, Filters};
+use crate::{measured_outcome, model_download, AppState, Filters, MeasuredFailure};
+use searchi_core::metrics::{MetricsMode, SearchResponse, TraceCollector};
 use searchi_core::SearchResult;
 use searchi_service::{EmbeddingReport, HybridSearch};
 use serde::Serialize;
@@ -24,10 +25,20 @@ pub struct SemanticState {
     downloaded_bytes: AtomicU64,
     error: Mutex<Option<String>>,
     service: Mutex<Option<HybridSearch>>,
+    media_tools: Option<searchi_media::FfmpegTools>,
+    media_setup_error: Option<String>,
 }
 
 impl SemanticState {
-    pub fn new(model_directory: PathBuf, worker: PathBuf) -> Self {
+    pub fn new(
+        model_directory: PathBuf,
+        worker: PathBuf,
+        media_tools: Result<searchi_media::FfmpegTools, String>,
+    ) -> Self {
+        let (media_tools, media_setup_error) = match media_tools {
+            Ok(tools) => (Some(tools), None),
+            Err(error) => (None, Some(error)),
+        };
         Self {
             model_directory,
             worker,
@@ -40,6 +51,8 @@ impl SemanticState {
             downloaded_bytes: AtomicU64::new(0),
             error: Mutex::new(None),
             service: Mutex::new(None),
+            media_tools,
+            media_setup_error,
         }
     }
 
@@ -118,6 +131,9 @@ pub struct SemanticStatus {
     pending: usize,
     warming: bool,
     worker_info: Option<searchi_service::WorkerInfo>,
+    media_tools_ready: bool,
+    media_setup_error: Option<String>,
+    media_failures: Vec<searchi_service::MediaFailure>,
 }
 
 struct WorkGuard<'a>(&'a AtomicBool);
@@ -158,6 +174,9 @@ pub fn semantic_status(state: State<'_, AppState>) -> Result<SemanticStatus, Str
             .try_lock()
             .ok()
             .and_then(|service| service.as_ref().and_then(HybridSearch::worker_info)),
+        media_tools_ready: semantic.media_tools.is_some(),
+        media_setup_error: semantic.media_setup_error.clone(),
+        media_failures: index.failures,
     })
 }
 
@@ -217,13 +236,42 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
         semantic.record_error(None);
         // Indexing uses its own isolated worker; interactive queries keep their own session.
         let result = HybridSearch::open(
-            database,
+            &database,
             &semantic.model_directory,
             &semantic.worker,
             threads,
         )
-        .and_then(|mut service| service.index_embeddings())
+        .and_then(|mut service| {
+            let tools = semantic.media_tools.as_ref().ok_or_else(|| {
+                anyhow::anyhow!("The media decoder is missing from this installation. Repair or reinstall searchi.")
+            })?;
+            service.index_embeddings_with_media(tools, &searchi_media::MediaLimits::default())
+        })
         .map_err(|error| error.to_string());
+        // Prepare the newly activated graph before advertising semantic readiness.
+        // Keyword search stays available while the cached query session catches up.
+        if result.is_ok() {
+            semantic.warmed.store(false, Ordering::Release);
+            semantic.warming.store(true, Ordering::Release);
+            let _warming_guard = WorkGuard(&semantic.warming);
+            let preparation = (|| {
+                let mut cached = semantic
+                    .service
+                    .lock()
+                    .map_err(|_| "Semantic session is unavailable.".to_string())?;
+                let service = cached
+                    .as_mut()
+                    .ok_or_else(|| "Semantic query session has not been prepared.".to_string())?;
+                service.warmup().map_err(|error| error.to_string())
+            })();
+            semantic.warmed.store(preparation.is_ok(), Ordering::Release);
+            if let Err(error) = preparation {
+                semantic.record_error(Some(format!(
+                    "The index was saved, but search preparation failed: {error}"
+                )));
+                return result;
+            }
+        }
         semantic.record_error(result.as_ref().err().cloned());
         result
     })
@@ -283,3 +331,85 @@ pub async fn hybrid_search(
     .await
     .map_err(|error| error.to_string())?
 }
+
+#[tauri::command]
+pub async fn hybrid_search_measured(
+    query: String,
+    filters: Option<Filters>,
+    debug_performance: bool,
+    state: State<'_, AppState>,
+) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
+    let started = std::time::Instant::now();
+    let mut trace = TraceCollector::new(if debug_performance {
+        MetricsMode::Debug
+    } else {
+        MetricsMode::Basic
+    });
+    let semantic = state.semantic.clone();
+    if !semantic.warmed.load(Ordering::Acquire) {
+        return Err(MeasuredFailure::new(
+            "Semantic search is preparing in the background; keyword results are available.",
+            trace,
+        ));
+    }
+    let threads = trace.timed("desktop.settings_lock", None, |_, _| {
+        state
+            .settings
+            .lock()
+            .map(|settings| settings.value.cpu_threads)
+            .map_err(|_| "Settings are unavailable.")
+    });
+    let threads = match threads {
+        Ok(threads) => threads,
+        Err(error) => return Err(MeasuredFailure::new(error, trace)),
+    };
+    if semantic
+        .querying
+        .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+        .is_err()
+    {
+        return Err(MeasuredFailure::new(
+            "Semantic worker is busy; keyword results remain available.",
+            trace,
+        ));
+    }
+    let database = state.database.clone();
+    let queue = trace.begin("desktop.worker_queue", None);
+    tauri::async_runtime::spawn_blocking(move || {
+        let _guard = WorkGuard(&semantic.querying);
+        trace.end(queue, Default::default());
+        let result = (|| {
+            let mut cached = trace.timed("desktop.semantic_lock", None, |_, _| {
+                semantic
+                    .service
+                    .lock()
+                    .map_err(|_| "Semantic session is unavailable.")
+            })?;
+            if cached.is_none() {
+                *cached = Some(trace.timed("desktop.session_initialize", None, |_, _| {
+                    HybridSearch::open(
+                        database,
+                        &semantic.model_directory,
+                        &semantic.worker,
+                        threads,
+                    )
+                    .map_err(|error| error.to_string())
+                })?);
+            }
+            cached
+                .as_mut()
+                .ok_or("Semantic session is unavailable.")?
+                .search_traced(
+                    &query,
+                    50,
+                    &filters.unwrap_or_default().into(),
+                    &mut trace,
+                    None,
+                )
+                .map_err(|error| error.to_string())
+        })();
+        measured_outcome(result, trace)
+    })
+    .await
+    .map_err(|error| MeasuredFailure::task(error.to_string(), started))?
+}