feat: add multimodal search and granular performance diagnostics

apps/desktop/src-tauri/src/main.rs

@@ -4,9 +4,11 @@ mod model_download;
 mod semantic_commands;
 mod settings;
 
+use searchi_core::metrics::{MetricsMode, SearchMetrics, SearchResponse, TraceCollector};
 use searchi_core::{IndexReport, IndexStats, SearchIndex, SearchOptions, SearchResult};
 use semantic_commands::{
-    hybrid_search, index_embeddings, semantic_status, setup_model, SemanticState,
+    hybrid_search, hybrid_search_measured, index_embeddings, semantic_status, setup_model,
+    SemanticState,
 };
 use serde::{Deserialize, Serialize};
 use settings::{Settings, SettingsStore};
@@ -31,6 +33,46 @@ struct AppState {
     semantic: Arc<SemanticState>,
 }
 
+#[derive(Serialize)]
+struct MeasuredFailure {
+    message: String,
+    metrics: SearchMetrics,
+}
+
+impl MeasuredFailure {
+    fn new(message: impl Into<String>, trace: TraceCollector) -> Self {
+        Self {
+            message: message.into(),
+            metrics: trace.finish(0),
+        }
+    }
+
+    fn task(message: impl Into<String>, started: std::time::Instant) -> Self {
+        Self {
+            message: message.into(),
+            metrics: SearchMetrics {
+                total_ms: started.elapsed().as_secs_f64() * 1000.0,
+                result_count: 0,
+                trace: None,
+                dropped_spans: 1,
+            },
+        }
+    }
+}
+
+fn measured_outcome(
+    result: Result<Vec<SearchResult>, String>,
+    trace: TraceCollector,
+) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
+    match result {
+        Ok(results) => Ok(SearchResponse {
+            metrics: trace.finish(results.len()),
+            results,
+        }),
+        Err(message) => Err(MeasuredFailure::new(message, trace)),
+    }
+}
+
 #[derive(Serialize)]
 struct DesktopStatus {
     settings: Settings,
@@ -78,6 +120,8 @@ struct Filters {
     extension: Option<String>,
     min_size: Option<u64>,
     max_size: Option<u64>,
+    min_duration_ms: Option<u32>,
+    max_duration_ms: Option<u32>,
     modified_after_unix_ms: Option<i64>,
     modified_before_unix_ms: Option<i64>,
 }
@@ -89,6 +133,8 @@ impl From<Filters> for SearchOptions {
             extension: value.extension,
             min_size: value.min_size,
             max_size: value.max_size,
+            min_duration_ms: value.min_duration_ms,
+            max_duration_ms: value.max_duration_ms,
             modified_after_unix_ms: value.modified_after_unix_ms,
             modified_before_unix_ms: value.modified_before_unix_ms,
         }
@@ -114,6 +160,46 @@ async fn search(
     .map_err(|e| e.to_string())?
 }
 
+/// Measured command preserves the legacy array-returning API for integrations.
+#[tauri::command]
+async fn search_measured(
+    query: String,
+    filters: Option<Filters>,
+    debug_performance: bool,
+    state: State<'_, AppState>,
+) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
+    let started = std::time::Instant::now();
+    let mut trace = TraceCollector::new(if debug_performance {
+        MetricsMode::Debug
+    } else {
+        MetricsMode::Basic
+    });
+    let queue = trace.begin("desktop.worker_queue", None);
+    let lexical = state.lexical.clone();
+    tauri::async_runtime::spawn_blocking(move || {
+        trace.end(queue, Default::default());
+        let result = (|| {
+            let index = trace.timed("desktop.lexical_lock", None, |_, _| {
+                lexical
+                    .try_lock()
+                    .map_err(|_| "Keyword worker is busy; try the latest query again.".to_owned())
+            })?;
+            index
+                .search_with_options_traced(
+                    &query,
+                    50,
+                    &filters.unwrap_or_default().into(),
+                    &mut trace,
+                    None,
+                )
+                .map_err(|error| error.to_string())
+        })();
+        measured_outcome(result, trace)
+    })
+    .await
+    .map_err(|error| MeasuredFailure::task(error.to_string(), started))?
+}
+
 #[tauri::command]
 async fn pick_folder(app: tauri::AppHandle) -> Result<Option<String>, String> {
     tauri::async_runtime::spawn_blocking(move || {
@@ -148,6 +234,68 @@ fn validated_result(index: &SearchIndex, path: &str) -> Result<PathBuf, String>
     Ok(canonical)
 }
 
+#[derive(Serialize)]
+struct MediaPreview {
+    path: String,
+    kind: &'static str,
+}
+
+// The asset protocol starts with an empty scope. Only a validated indexed
+// media file is granted access; the webview receives no general filesystem API.
+#[tauri::command]
+async fn prepare_media_preview(
+    path: String,
+    app: tauri::AppHandle,
+    state: State<'_, AppState>,
+) -> Result<MediaPreview, String> {
+    let database = state.database.clone();
+    tauri::async_runtime::spawn_blocking(move || {
+        let index = SearchIndex::open(database).map_err(|error| error.to_string())?;
+        let canonical = validated_result(&index, &path)?;
+        let kind = match searchi_media::classify(&canonical) {
+            Some(searchi_media::MediaKind::Image) => "image",
+            Some(searchi_media::MediaKind::Audio) => "audio",
+            Some(searchi_media::MediaKind::Video) => "video",
+            None => return Err("This file is not a supported media preview.".into()),
+        };
+        if kind == "image" {
+            let bytes = std::fs::metadata(&canonical)
+                .map_err(|error| error.to_string())?
+                .len();
+            if bytes > 64 * 1024 * 1024 {
+                return Err("Image exceeds the 64 MiB preview limit.".into());
+            }
+            let (width, height) = image::ImageReader::open(&canonical)
+                .map_err(|error| error.to_string())?
+                .with_guessed_format()
+                .map_err(|error| error.to_string())?
+                .into_dimensions()
+                .map_err(|error| format!("Image preview is unavailable: {error}"))?;
+            let jpeg = canonical
+                .extension()
+                .and_then(|value| value.to_str())
+                .is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "jpg" | "jpeg"));
+            let max_pixels = if jpeg {
+                64 * 1024 * 1024
+            } else {
+                16 * 1024 * 1024
+            };
+            if width == 0 || height == 0 || u64::from(width) * u64::from(height) > max_pixels {
+                return Err("Image exceeds the supported preview dimensions.".into());
+            }
+        }
+        app.asset_protocol_scope()
+            .allow_file(&canonical)
+            .map_err(|error| error.to_string())?;
+        Ok(MediaPreview {
+            path: canonical.to_string_lossy().into_owned(),
+            kind,
+        })
+    })
+    .await
+    .map_err(|error| error.to_string())?
+}
+
 #[tauri::command]
 async fn open_result(
     path: String,
@@ -279,7 +427,24 @@ fn main() {
                         app.path().resource_dir().unwrap_or_else(|_| directory.clone()).join(worker_name)
                     }
                 });
-            let semantic = Arc::new(SemanticState::new(model_directory, worker));
+            let media_tools = (|| -> Result<searchi_media::FfmpegTools, String> {
+                if let Some(tools) = searchi_media::FfmpegTools::from_environment()
+                    .map_err(|error| error.to_string())? {
+                    return Ok(tools);
+                }
+                let development_media = development_root.join(".searchi/media-runtime/bin");
+                let media_directory = if cfg!(debug_assertions) && development_media.is_dir() {
+                    development_media
+                } else {
+                    app.path().resource_dir().map_err(|error| error.to_string())?.join("media-runtime/bin")
+                };
+                let ffmpeg = media_directory.join(if cfg!(windows) { "ffmpeg.exe" } else { "ffmpeg" });
+                let ffprobe = media_directory.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" });
+                searchi_media::FfmpegTools::new(ffmpeg, ffprobe).map_err(|_| {
+                    "The media decoder is missing from this installation. Repair or reinstall searchi.".into()
+                })
+            })();
+            let semantic = Arc::new(SemanticState::new(model_directory, worker, media_tools));
             semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
             #[cfg(target_os = "macos")]
             let shortcut = "Shift+Super+Space";
@@ -307,16 +472,19 @@ fn main() {
         })
         .invoke_handler(tauri::generate_handler![
             search,
+            search_measured,
             stats,
             index_folder,
             pick_folder,
             open_result,
+            prepare_media_preview,
             desktop_status,
             set_cpu_threads,
             semantic_status,
             setup_model,
             index_embeddings,
-            hybrid_search
+            hybrid_search,
+            hybrid_search_measured
         ])
         .run(tauri::generate_context!())
         .expect("Searchi desktop could not start");
@@ -326,6 +494,32 @@ fn main() {
 mod tests {
     use super::*;
 
+    #[test]
+    fn measured_failure_retains_closed_trace_without_inventing_results() {
+        let mut trace = TraceCollector::new(MetricsMode::Debug);
+        let parent = trace.begin("desktop.request", None);
+        trace.begin("desktop.lexical_lock", parent);
+        let failed = measured_outcome(Err("Keyword worker is busy".into()), trace)
+            .err()
+            .unwrap();
+        assert_eq!(failed.message, "Keyword worker is busy");
+        assert_eq!(failed.metrics.result_count, 0);
+        let spans = failed.metrics.trace.unwrap();
+        assert_eq!(spans.len(), 2);
+        assert_eq!(spans[1].parent, Some(0));
+        assert!(spans[1].start_ms + spans[1].duration_ms <= failed.metrics.total_ms + 0.001);
+    }
+
+    #[test]
+    fn measured_empty_success_is_distinct_from_failure_and_basic_has_no_trace() {
+        let response = measured_outcome(Ok(Vec::new()), TraceCollector::new(MetricsMode::Basic))
+            .ok()
+            .unwrap();
+        assert!(response.results.is_empty());
+        assert_eq!(response.metrics.result_count, 0);
+        assert!(response.metrics.trace.is_none());
+    }
+
     #[test]
     fn indexing_gate_is_released_when_worker_unwinds() {
         let indexing = Arc::new(AtomicBool::new(true));

apps/desktop/src-tauri/src/semantic_commands.rs

@@ -1,5 +1,6 @@
 //! Desktop-owned orchestration: managed model setup and persistent hybrid queries.
-use crate::{model_download, AppState, Filters};
+use crate::{measured_outcome, model_download, AppState, Filters, MeasuredFailure};
+use searchi_core::metrics::{MetricsMode, SearchResponse, TraceCollector};
 use searchi_core::SearchResult;
 use searchi_service::{EmbeddingReport, HybridSearch};
 use serde::Serialize;
@@ -24,10 +25,20 @@ pub struct SemanticState {
     downloaded_bytes: AtomicU64,
     error: Mutex<Option<String>>,
     service: Mutex<Option<HybridSearch>>,
+    media_tools: Option<searchi_media::FfmpegTools>,
+    media_setup_error: Option<String>,
 }
 
 impl SemanticState {
-    pub fn new(model_directory: PathBuf, worker: PathBuf) -> Self {
+    pub fn new(
+        model_directory: PathBuf,
+        worker: PathBuf,
+        media_tools: Result<searchi_media::FfmpegTools, String>,
+    ) -> Self {
+        let (media_tools, media_setup_error) = match media_tools {
+            Ok(tools) => (Some(tools), None),
+            Err(error) => (None, Some(error)),
+        };
         Self {
             model_directory,
             worker,
@@ -40,6 +51,8 @@ impl SemanticState {
             downloaded_bytes: AtomicU64::new(0),
             error: Mutex::new(None),
             service: Mutex::new(None),
+            media_tools,
+            media_setup_error,
         }
     }
 
@@ -118,6 +131,9 @@ pub struct SemanticStatus {
     pending: usize,
     warming: bool,
     worker_info: Option<searchi_service::WorkerInfo>,
+    media_tools_ready: bool,
+    media_setup_error: Option<String>,
+    media_failures: Vec<searchi_service::MediaFailure>,
 }
 
 struct WorkGuard<'a>(&'a AtomicBool);
@@ -158,6 +174,9 @@ pub fn semantic_status(state: State<'_, AppState>) -> Result<SemanticStatus, Str
             .try_lock()
             .ok()
             .and_then(|service| service.as_ref().and_then(HybridSearch::worker_info)),
+        media_tools_ready: semantic.media_tools.is_some(),
+        media_setup_error: semantic.media_setup_error.clone(),
+        media_failures: index.failures,
     })
 }
 
@@ -217,13 +236,42 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
         semantic.record_error(None);
         // Indexing uses its own isolated worker; interactive queries keep their own session.
         let result = HybridSearch::open(
-            database,
+            &database,
             &semantic.model_directory,
             &semantic.worker,
             threads,
         )
-        .and_then(|mut service| service.index_embeddings())
+        .and_then(|mut service| {
+            let tools = semantic.media_tools.as_ref().ok_or_else(|| {
+                anyhow::anyhow!("The media decoder is missing from this installation. Repair or reinstall searchi.")
+            })?;
+            service.index_embeddings_with_media(tools, &searchi_media::MediaLimits::default())
+        })
         .map_err(|error| error.to_string());
+        // Prepare the newly activated graph before advertising semantic readiness.
+        // Keyword search stays available while the cached query session catches up.
+        if result.is_ok() {
+            semantic.warmed.store(false, Ordering::Release);
+            semantic.warming.store(true, Ordering::Release);
+            let _warming_guard = WorkGuard(&semantic.warming);
+            let preparation = (|| {
+                let mut cached = semantic
+                    .service
+                    .lock()
+                    .map_err(|_| "Semantic session is unavailable.".to_string())?;
+                let service = cached
+                    .as_mut()
+                    .ok_or_else(|| "Semantic query session has not been prepared.".to_string())?;
+                service.warmup().map_err(|error| error.to_string())
+            })();
+            semantic.warmed.store(preparation.is_ok(), Ordering::Release);
+            if let Err(error) = preparation {
+                semantic.record_error(Some(format!(
+                    "The index was saved, but search preparation failed: {error}"
+                )));
+                return result;
+            }
+        }
         semantic.record_error(result.as_ref().err().cloned());
         result
     })
@@ -283,3 +331,85 @@ pub async fn hybrid_search(
     .await
     .map_err(|error| error.to_string())?
 }
+
+#[tauri::command]
+pub async fn hybrid_search_measured(
+    query: String,
+    filters: Option<Filters>,
+    debug_performance: bool,
+    state: State<'_, AppState>,
+) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
+    let started = std::time::Instant::now();
+    let mut trace = TraceCollector::new(if debug_performance {
+        MetricsMode::Debug
+    } else {
+        MetricsMode::Basic
+    });
+    let semantic = state.semantic.clone();
+    if !semantic.warmed.load(Ordering::Acquire) {
+        return Err(MeasuredFailure::new(
+            "Semantic search is preparing in the background; keyword results are available.",
+            trace,
+        ));
+    }
+    let threads = trace.timed("desktop.settings_lock", None, |_, _| {
+        state
+            .settings
+            .lock()
+            .map(|settings| settings.value.cpu_threads)
+            .map_err(|_| "Settings are unavailable.")
+    });
+    let threads = match threads {
+        Ok(threads) => threads,
+        Err(error) => return Err(MeasuredFailure::new(error, trace)),
+    };
+    if semantic
+        .querying
+        .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+        .is_err()
+    {
+        return Err(MeasuredFailure::new(
+            "Semantic worker is busy; keyword results remain available.",
+            trace,
+        ));
+    }
+    let database = state.database.clone();
+    let queue = trace.begin("desktop.worker_queue", None);
+    tauri::async_runtime::spawn_blocking(move || {
+        let _guard = WorkGuard(&semantic.querying);
+        trace.end(queue, Default::default());
+        let result = (|| {
+            let mut cached = trace.timed("desktop.semantic_lock", None, |_, _| {
+                semantic
+                    .service
+                    .lock()
+                    .map_err(|_| "Semantic session is unavailable.")
+            })?;
+            if cached.is_none() {
+                *cached = Some(trace.timed("desktop.session_initialize", None, |_, _| {
+                    HybridSearch::open(
+                        database,
+                        &semantic.model_directory,
+                        &semantic.worker,
+                        threads,
+                    )
+                    .map_err(|error| error.to_string())
+                })?);
+            }
+            cached
+                .as_mut()
+                .ok_or("Semantic session is unavailable.")?
+                .search_traced(
+                    &query,
+                    50,
+                    &filters.unwrap_or_default().into(),
+                    &mut trace,
+                    None,
+                )
+                .map_err(|error| error.to_string())
+        })();
+        measured_outcome(result, trace)
+    })
+    .await
+    .map_err(|error| MeasuredFailure::task(error.to_string(), started))?
+}