Add local spoken passages with Google retrieval and isolated runtime setup

apps/desktop/src-tauri/src/main.rs

@@ -11,6 +11,7 @@ mod media_preview;
 mod model_download;
 mod semantic_commands;
 mod settings;
+mod speech_setup;
 mod watching;
 
 use ai_commands::{generate_ai, prepare_ai_source, AiSession};
@@ -650,7 +651,18 @@ fn main() {
                     "The media decoder is missing from this installation. Repair or reinstall searchi.".into()
                 })
             })();
+            let speech_worker = std::env::var_os("SEARCHI_SPEECH_WORKER").map(PathBuf::from).unwrap_or_else(|| {
+                            let backend = if std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1") { "cpu" } else { "auto" };
+                            let mut development = development_root.join(".searchi/speech-runtime").join(backend);
+                            if cfg!(windows) { development = development.join("Release"); }
+                            let name = if cfg!(windows) { "searchi-speech-worker.exe" } else { "searchi-speech-worker" };
+                            let development = development.join(name);
+                            if cfg!(debug_assertions) && development.is_file() { development } else {
+                                app.path().resource_dir().unwrap_or_else(|_| directory.clone()).join("speech-runtime").join(name)
+                            }
+                        });
             let mut ignored = vec![directory.clone(), model_directory.clone()];
+            if let Some(parent) = speech_worker.parent().filter(|path| path.is_absolute()) { ignored.push(parent.to_path_buf()); }
             if let Some(parent) = worker.parent().filter(|path| path.is_absolute()) {
                 ignored.push(parent.to_path_buf());
             }
@@ -667,7 +679,9 @@ fn main() {
             ).map_err(std::io::Error::other)?);
             let semantic = Arc::new(
                 SemanticState::new(model_directory, worker, media_tools)
-                    .with_media_scratch(directory.join("MediaScratch")),
+                    .with_media_scratch(directory.join("MediaScratch"))
+                    .with_speech_assets(
+speech_worker, directory.join("SpeechModels")),
             );
             semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
             #[cfg(target_os = "macos")]

apps/desktop/src-tauri/src/semantic_commands.rs

@@ -18,6 +18,7 @@ use tauri::State;
 pub struct SemanticState {
     pub model_directory: PathBuf,
     pub worker: PathBuf,
+    speech: std::sync::Arc<crate::speech_setup::SpeechSetup>,
     verified: AtomicBool,
     text_verified: AtomicBool,
     media_verified: AtomicBool,
@@ -48,7 +49,23 @@ impl SemanticState {
             Ok(tools) => (Some(tools), None),
             Err(error) => (None, Some(error)),
         };
+        let speech = crate::speech_setup::SpeechSetup::new(
+            worker
+                .parent()
+                .unwrap_or_else(|| std::path::Path::new("."))
+                .join("speech-runtime")
+                .join(if cfg!(windows) {
+                    "searchi-speech-worker.exe"
+                } else {
+                    "searchi-speech-worker"
+                }),
+            model_directory
+                .parent()
+                .unwrap_or_else(|| std::path::Path::new("."))
+                .join("speech"),
+        );
         Self {
+            speech,
             model_directory,
             worker,
             verified: AtomicBool::new(false),
@@ -72,6 +89,11 @@ impl SemanticState {
         }
     }
 
+    pub fn with_speech_assets(mut self, worker: PathBuf, models: PathBuf) -> Self {
+        self.speech = crate::speech_setup::SpeechSetup::new(worker, models);
+        self
+    }
+
     /// Keep generated video frames below the application's excluded data tree.
     pub fn with_media_scratch(mut self, directory: PathBuf) -> Self {
         self.media_limits.scratch_directory = Some(directory);
@@ -222,7 +244,12 @@ impl SemanticState {
             })
             .unwrap_or(Ok(()));
         self.warming.store(false, Ordering::Release);
-        self.finish_setup(provision, prepared)
+        let result = self.finish_setup(provision, prepared);
+        if result.is_ok() {
+            self.speech
+                .start(self.inference_budget.clone(), self.media_limits.clone());
+        }
+        result
     }
 
     fn finish_setup(
@@ -299,6 +326,7 @@ impl SemanticState {
 
 #[derive(Serialize)]
 pub struct SemanticStatus {
+    speech: crate::speech_setup::SpeechStatus,
     model: &'static str,
     ready: bool,
     text_model_ready: bool,
@@ -340,8 +368,11 @@ pub fn semantic_status(state: State<'_, AppState>) -> Result<SemanticStatus, Str
     let desired = semantic.desired_coverage();
     let index = HybridSearch::embedding_status_for_coverage(&state.database, desired)
         .map_err(|error| error.to_string())?;
-    let indexing_ready = semantic.can_index(index.text_indexing_allowed);
+    let speech = semantic.speech.status(&index);
+    let indexing_ready = semantic.can_index(index.text_indexing_allowed)
+        && (index.speech_policy.is_none() || speech.ready);
     Ok(SemanticStatus {
+        speech,
         model: "Google EmbeddingGemma 2",
         ready: worker_ready && semantic.warmed.load(Ordering::Acquire),
         text_model_ready: semantic.text_verified.load(Ordering::Acquire),
@@ -428,7 +459,9 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
     let coverage = semantic.desired_coverage();
     let index = HybridSearch::embedding_status_for_coverage(&state.database, coverage)
         .map_err(|error| error.to_string())?;
-    if !semantic.can_index(index.text_indexing_allowed) {
+    if !semantic.can_index(index.text_indexing_allowed)
+        || (index.speech_policy.is_some() && !semantic.speech.status(&index).ready)
+    {
         return Err(
             "Semantic indexing is waiting for verified models and query preparation.".into(),
         );
@@ -464,6 +497,14 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
                     .and_then(|report| report.ok_or_else(|| anyhow::anyhow!("Full media index retained while media setup finishes."))),
                 SemanticCoverage::FullMedia => {
                     let tools = semantic.media_tools.as_ref().ok_or_else(|| anyhow::anyhow!("The media decoder is missing from this installation. Repair or reinstall searchi."))?;
+                    let admission = semantic.inference_budget.acquire(InferenceRole::Background, 1)?;
+                    let speech = semantic.speech.worker_for_index()?;
+                    drop(admission);
+                    if let Some(speech) = speech {
+                        service = service.with_speech_provider(Box::new(searchi_service::BudgetedSpeechProvider::new(
+                            Box::new(speech), semantic.inference_budget.clone(),
+                        )))?;
+                    }
                     service.index_embeddings_with_media(tools, &semantic.media_limits)
                 }
             }

apps/desktop/src-tauri/src/speech_setup.rs

@@ -0,0 +1,120 @@
+//! Supplemental local speech setup; Google search readiness stays independent.
+use searchi_service::{EmbeddingStatus, InferenceBudget, InferenceRole, MediaLimits};
+use searchi_speech::{Device, SpeechWorker};
+use serde::Serialize;
+use std::{
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicBool, AtomicU64, Ordering},
+        Arc, Mutex,
+    },
+};
+
+pub struct SpeechSetup {
+    worker: PathBuf,
+    models: PathBuf,
+    downloading: AtomicBool,
+    downloaded: AtomicU64,
+    ready: AtomicBool,
+    error: Mutex<Option<String>>,
+    policy: Mutex<Option<String>>,
+    prepared: Mutex<Option<SpeechWorker>>,
+}
+
+#[derive(Serialize)]
+pub struct SpeechStatus {
+    worker_ready: bool,
+    pub ready: bool,
+    downloading: bool,
+    downloaded_bytes: u64,
+    total_bytes: u64,
+    error: Option<String>,
+    requested_policy: Option<String>,
+    pub pending: bool,
+    corpus_revision: i64,
+    failures: usize,
+}
+
+impl SpeechSetup {
+    pub fn new(worker: PathBuf, models: PathBuf) -> Arc<Self> {
+        Arc::new(Self {
+            worker,
+            models,
+            downloading: AtomicBool::new(false),
+            downloaded: AtomicU64::new(0),
+            ready: AtomicBool::new(false),
+            error: Mutex::new(None),
+            policy: Mutex::new(None),
+            prepared: Mutex::new(None),
+        })
+    }
+
+    pub fn start(self: &Arc<Self>, budget: InferenceBudget, limits: MediaLimits) {
+        if self.downloading.swap(true, Ordering::AcqRel) {
+            return;
+        }
+        let setup = self.clone();
+        let spawned = std::thread::Builder::new().name("searchi-speech-setup".into()).spawn(move || {
+            let result = (|| -> anyhow::Result<()> {
+                anyhow::ensure!(setup.worker.is_file(), "Speech recognition is unavailable in this installation. Repair searchi to enable it.");
+                searchi_assets::speech::provision(&setup.models, &setup.downloaded)?;
+                let _admission = budget.acquire(InferenceRole::Background, 1)?;
+                let worker = setup.load()?;
+                let policy = searchi_service::speech_policy_key(worker.fingerprint(), &limits)?;
+                *setup.prepared.lock().map_err(|_| anyhow::anyhow!("Speech setup unavailable"))? = Some(worker);
+                *setup.policy.lock().map_err(|_| anyhow::anyhow!("Speech setup unavailable"))? = Some(policy);
+                Ok(())
+            })();
+            setup.ready.store(result.is_ok(), Ordering::Release);
+            if let Ok(mut error) = setup.error.lock() { *error = result.err().map(|error| error.to_string()); }
+            setup.downloading.store(false, Ordering::Release);
+        });
+        if let Err(error) = spawned {
+            if let Ok(mut last) = self.error.lock() {
+                *last = Some(format!("Could not start speech setup: {error}"));
+            }
+            self.downloading.store(false, Ordering::Release);
+        }
+    }
+
+    fn load(&self) -> anyhow::Result<SpeechWorker> {
+        SpeechWorker::load(
+            &self.worker,
+            &self.models.join("ggml-base.bin"),
+            &self.models.join("ggml-silero-v6.2.0.bin"),
+            Device::Auto,
+        )
+    }
+
+    pub fn worker_for_index(&self) -> anyhow::Result<Option<SpeechWorker>> {
+        if !self.ready.load(Ordering::Acquire) {
+            return Ok(None);
+        }
+        if let Some(worker) = self
+            .prepared
+            .lock()
+            .map_err(|_| anyhow::anyhow!("Speech setup unavailable"))?
+            .take()
+        {
+            return Ok(Some(worker));
+        }
+        self.load().map(Some)
+    }
+
+    pub fn status(&self, index: &EmbeddingStatus) -> SpeechStatus {
+        let policy = self.policy.lock().ok().and_then(|value| value.clone());
+        let ready = self.ready.load(Ordering::Acquire);
+        SpeechStatus {
+            worker_ready: self.worker.is_file(),
+            ready,
+            downloading: self.downloading.load(Ordering::Acquire),
+            downloaded_bytes: self.downloaded.load(Ordering::Acquire),
+            total_bytes: searchi_assets::speech::TOTAL_BYTES,
+            error: self.error.lock().ok().and_then(|value| value.clone()),
+            pending: ready && (index.speech_policy != policy || !index.speech_failures.is_empty()),
+            requested_policy: policy,
+            corpus_revision: index.corpus_revision,
+            failures: index.speech_failures.len(),
+        }
+    }
+}