feat: add multimodal search and granular performance diagnostics

apps/desktop/src-tauri/src/main.rs

@@ -4,9 +4,11 @@ mod model_download;
 mod semantic_commands;
 mod settings;
 
+use searchi_core::metrics::{MetricsMode, SearchMetrics, SearchResponse, TraceCollector};
 use searchi_core::{IndexReport, IndexStats, SearchIndex, SearchOptions, SearchResult};
 use semantic_commands::{
-    hybrid_search, index_embeddings, semantic_status, setup_model, SemanticState,
+    hybrid_search, hybrid_search_measured, index_embeddings, semantic_status, setup_model,
+    SemanticState,
 };
 use serde::{Deserialize, Serialize};
 use settings::{Settings, SettingsStore};
@@ -31,6 +33,46 @@ struct AppState {
     semantic: Arc<SemanticState>,
 }
 
+#[derive(Serialize)]
+struct MeasuredFailure {
+    message: String,
+    metrics: SearchMetrics,
+}
+
+impl MeasuredFailure {
+    fn new(message: impl Into<String>, trace: TraceCollector) -> Self {
+        Self {
+            message: message.into(),
+            metrics: trace.finish(0),
+        }
+    }
+
+    fn task(message: impl Into<String>, started: std::time::Instant) -> Self {
+        Self {
+            message: message.into(),
+            metrics: SearchMetrics {
+                total_ms: started.elapsed().as_secs_f64() * 1000.0,
+                result_count: 0,
+                trace: None,
+                dropped_spans: 1,
+            },
+        }
+    }
+}
+
+fn measured_outcome(
+    result: Result<Vec<SearchResult>, String>,
+    trace: TraceCollector,
+) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
+    match result {
+        Ok(results) => Ok(SearchResponse {
+            metrics: trace.finish(results.len()),
+            results,
+        }),
+        Err(message) => Err(MeasuredFailure::new(message, trace)),
+    }
+}
+
 #[derive(Serialize)]
 struct DesktopStatus {
     settings: Settings,
@@ -78,6 +120,8 @@ struct Filters {
     extension: Option<String>,
     min_size: Option<u64>,
     max_size: Option<u64>,
+    min_duration_ms: Option<u32>,
+    max_duration_ms: Option<u32>,
     modified_after_unix_ms: Option<i64>,
     modified_before_unix_ms: Option<i64>,
 }
@@ -89,6 +133,8 @@ impl From<Filters> for SearchOptions {
             extension: value.extension,
             min_size: value.min_size,
             max_size: value.max_size,
+            min_duration_ms: value.min_duration_ms,
+            max_duration_ms: value.max_duration_ms,
             modified_after_unix_ms: value.modified_after_unix_ms,
             modified_before_unix_ms: value.modified_before_unix_ms,
         }
@@ -114,6 +160,46 @@ async fn search(
     .map_err(|e| e.to_string())?
 }
 
+/// Measured command preserves the legacy array-returning API for integrations.
+#[tauri::command]
+async fn search_measured(
+    query: String,
+    filters: Option<Filters>,
+    debug_performance: bool,
+    state: State<'_, AppState>,
+) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
+    let started = std::time::Instant::now();
+    let mut trace = TraceCollector::new(if debug_performance {
+        MetricsMode::Debug
+    } else {
+        MetricsMode::Basic
+    });
+    let queue = trace.begin("desktop.worker_queue", None);
+    let lexical = state.lexical.clone();
+    tauri::async_runtime::spawn_blocking(move || {
+        trace.end(queue, Default::default());
+        let result = (|| {
+            let index = trace.timed("desktop.lexical_lock", None, |_, _| {
+                lexical
+                    .try_lock()
+                    .map_err(|_| "Keyword worker is busy; try the latest query again.".to_owned())
+            })?;
+            index
+                .search_with_options_traced(
+                    &query,
+                    50,
+                    &filters.unwrap_or_default().into(),
+                    &mut trace,
+                    None,
+                )
+                .map_err(|error| error.to_string())
+        })();
+        measured_outcome(result, trace)
+    })
+    .await
+    .map_err(|error| MeasuredFailure::task(error.to_string(), started))?
+}
+
 #[tauri::command]
 async fn pick_folder(app: tauri::AppHandle) -> Result<Option<String>, String> {
     tauri::async_runtime::spawn_blocking(move || {
@@ -148,6 +234,68 @@ fn validated_result(index: &SearchIndex, path: &str) -> Result<PathBuf, String>
     Ok(canonical)
 }
 
+#[derive(Serialize)]
+struct MediaPreview {
+    path: String,
+    kind: &'static str,
+}
+
+// The asset protocol starts with an empty scope. Only a validated indexed
+// media file is granted access; the webview receives no general filesystem API.
+#[tauri::command]
+async fn prepare_media_preview(
+    path: String,
+    app: tauri::AppHandle,
+    state: State<'_, AppState>,
+) -> Result<MediaPreview, String> {
+    let database = state.database.clone();
+    tauri::async_runtime::spawn_blocking(move || {
+        let index = SearchIndex::open(database).map_err(|error| error.to_string())?;
+        let canonical = validated_result(&index, &path)?;
+        let kind = match searchi_media::classify(&canonical) {
+            Some(searchi_media::MediaKind::Image) => "image",
+            Some(searchi_media::MediaKind::Audio) => "audio",
+            Some(searchi_media::MediaKind::Video) => "video",
+            None => return Err("This file is not a supported media preview.".into()),
+        };
+        if kind == "image" {
+            let bytes = std::fs::metadata(&canonical)
+                .map_err(|error| error.to_string())?
+                .len();
+            if bytes > 64 * 1024 * 1024 {
+                return Err("Image exceeds the 64 MiB preview limit.".into());
+            }
+            let (width, height) = image::ImageReader::open(&canonical)
+                .map_err(|error| error.to_string())?
+                .with_guessed_format()
+                .map_err(|error| error.to_string())?
+                .into_dimensions()
+                .map_err(|error| format!("Image preview is unavailable: {error}"))?;
+            let jpeg = canonical
+                .extension()
+                .and_then(|value| value.to_str())
+                .is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "jpg" | "jpeg"));
+            let max_pixels = if jpeg {
+                64 * 1024 * 1024
+            } else {
+                16 * 1024 * 1024
+            };
+            if width == 0 || height == 0 || u64::from(width) * u64::from(height) > max_pixels {
+                return Err("Image exceeds the supported preview dimensions.".into());
+            }
+        }
+        app.asset_protocol_scope()
+            .allow_file(&canonical)
+            .map_err(|error| error.to_string())?;
+        Ok(MediaPreview {
+            path: canonical.to_string_lossy().into_owned(),
+            kind,
+        })
+    })
+    .await
+    .map_err(|error| error.to_string())?
+}
+
 #[tauri::command]
 async fn open_result(
     path: String,
@@ -279,7 +427,24 @@ fn main() {
                         app.path().resource_dir().unwrap_or_else(|_| directory.clone()).join(worker_name)
                     }
                 });
-            let semantic = Arc::new(SemanticState::new(model_directory, worker));
+            let media_tools = (|| -> Result<searchi_media::FfmpegTools, String> {
+                if let Some(tools) = searchi_media::FfmpegTools::from_environment()
+                    .map_err(|error| error.to_string())? {
+                    return Ok(tools);
+                }
+                let development_media = development_root.join(".searchi/media-runtime/bin");
+                let media_directory = if cfg!(debug_assertions) && development_media.is_dir() {
+                    development_media
+                } else {
+                    app.path().resource_dir().map_err(|error| error.to_string())?.join("media-runtime/bin")
+                };
+                let ffmpeg = media_directory.join(if cfg!(windows) { "ffmpeg.exe" } else { "ffmpeg" });
+                let ffprobe = media_directory.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" });
+                searchi_media::FfmpegTools::new(ffmpeg, ffprobe).map_err(|_| {
+                    "The media decoder is missing from this installation. Repair or reinstall searchi.".into()
+                })
+            })();
+            let semantic = Arc::new(SemanticState::new(model_directory, worker, media_tools));
             semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
             #[cfg(target_os = "macos")]
             let shortcut = "Shift+Super+Space";
@@ -307,16 +472,19 @@ fn main() {
         })
         .invoke_handler(tauri::generate_handler![
             search,
+            search_measured,
             stats,
             index_folder,
             pick_folder,
             open_result,
+            prepare_media_preview,
             desktop_status,
             set_cpu_threads,
             semantic_status,
             setup_model,
             index_embeddings,
-            hybrid_search
+            hybrid_search,
+            hybrid_search_measured
         ])
         .run(tauri::generate_context!())
         .expect("Searchi desktop could not start");
@@ -326,6 +494,32 @@ fn main() {
 mod tests {
     use super::*;
 
+    #[test]
+    fn measured_failure_retains_closed_trace_without_inventing_results() {
+        let mut trace = TraceCollector::new(MetricsMode::Debug);
+        let parent = trace.begin("desktop.request", None);
+        trace.begin("desktop.lexical_lock", parent);
+        let failed = measured_outcome(Err("Keyword worker is busy".into()), trace)
+            .err()
+            .unwrap();
+        assert_eq!(failed.message, "Keyword worker is busy");
+        assert_eq!(failed.metrics.result_count, 0);
+        let spans = failed.metrics.trace.unwrap();
+        assert_eq!(spans.len(), 2);
+        assert_eq!(spans[1].parent, Some(0));
+        assert!(spans[1].start_ms + spans[1].duration_ms <= failed.metrics.total_ms + 0.001);
+    }
+
+    #[test]
+    fn measured_empty_success_is_distinct_from_failure_and_basic_has_no_trace() {
+        let response = measured_outcome(Ok(Vec::new()), TraceCollector::new(MetricsMode::Basic))
+            .ok()
+            .unwrap();
+        assert!(response.results.is_empty());
+        assert_eq!(response.metrics.result_count, 0);
+        assert!(response.metrics.trace.is_none());
+    }
+
     #[test]
     fn indexing_gate_is_released_when_worker_unwinds() {
         let indexing = Arc::new(AtomicBool::new(true));