feat: add multimodal search and granular performance diagnostics

tools/media-eval/src/bin/image_budgets.rs

@@ -0,0 +1,100 @@
+//! Same frozen real photos/queries across predeclared token budgets.
+use anyhow::{ensure, Result};
+use searchi_semantic::{Device, GemmaEmbedder, GEMMA_MEDIA_FINGERPRINT};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 4,
+        "Supply CORPUS_ROOT MODEL_DIR CPU_WORKER OUTPUT_JSON"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU only"
+    );
+    let root = PathBuf::from(&args[0]);
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(lock["fixture_sha256"] == hash, "Frozen labels changed");
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    let photos: Vec<_> = fixture["assets"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|asset| asset["kind"] == "image")
+        .collect();
+    let queries: Vec<_> = fixture["queries"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|query| query["category"] == "image-memory")
+        .collect();
+    for photo in &photos {
+        let content = fs::read(root.join(photo["path"].as_str().unwrap()))?;
+        ensure!(
+            photo["sha256"] == format!("{:x}", Sha256::digest(content)),
+            "Frozen photo changed"
+        );
+    }
+    let mut model = GemmaEmbedder::load_with_device(&args[1], &args[2], 1, Device::Cpu)?;
+    let mut query_vectors = Vec::new();
+    let query_start = Instant::now();
+    for query in &queries {
+        query_vectors.push(model.embed_query(query["text"].as_str().unwrap())?);
+    }
+    let query_embedding_ms = query_start.elapsed().as_secs_f64() * 1000.0;
+    let mut budgets = Vec::new();
+    for budget in [70, 280, 1120] {
+        let start = Instant::now();
+        let mut encoded = Vec::new();
+        for photo in &photos {
+            let path = root.join(photo["path"].as_str().unwrap());
+            let image = model
+                .embed_images_with_metadata(&[&path], budget)?
+                .remove(0);
+            encoded.push(image);
+        }
+        let encoding_ms = start.elapsed().as_secs_f64() * 1000.0;
+        let mut rows = Vec::new();
+        for (query, query_vector) in queries.iter().zip(&query_vectors) {
+            let rank_start = Instant::now();
+            let mut ranked: Vec<_> = photos
+                .iter()
+                .zip(&encoded)
+                .map(|(photo, image)| {
+                    let score: f64 = query_vector
+                        .iter()
+                        .zip(&image.vector)
+                        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+                        .sum();
+                    json!({"candidate":{"asset":photo["id"],"modality":"image"},"score":score})
+                })
+                .collect();
+            ranked.sort_by(|a, b| {
+                b["score"]
+                    .as_f64()
+                    .unwrap()
+                    .total_cmp(&a["score"].as_f64().unwrap())
+            });
+            rows.push(json!({"query":query,"file_rankings":ranked,"segment_top5":[],"ranking_only_ms":rank_start.elapsed().as_secs_f64()*1000.0}));
+        }
+        let resolutions: Vec<_> = photos.iter().zip(&encoded).map(|(photo,image)| json!({"asset":photo["id"],"encoded_width":image.encoded_width,"encoded_height":image.encoded_height,"token_budget":image.token_budget})).collect();
+        budgets.push(json!({"token_budget":budget,"encoding_ms":encoding_ms,"images":resolutions,"rows":rows}));
+        println!(
+            "Budget {budget}: {} actual photo encodings, {} frozen queries",
+            photos.len(),
+            queries.len()
+        );
+    }
+    fs::write(
+        &args[3],
+        serde_json::to_vec_pretty(
+            &json!({"fixture_sha256":hash,"worker":model.info,"media_fingerprint":GEMMA_MEDIA_FINGERPRINT,"photos":photos.len(),"queries":queries.len(),"query_embedding_ms":query_embedding_ms,"budgets":budgets,"scope":"12-photo-only token-budget comparison; same 24 independently frozen queries, query embeddings computed once and shared across budgets; ranking_only_ms excludes inference; no retrieval threshold or label tuning. Does not certify production search."}),
+        )?,
+    )?;
+    Ok(())
+}

tools/media-eval/src/bin/metadata.rs

@@ -0,0 +1,78 @@
+//! Fast actual metadata/keyword check, separate from embedding semantics.
+use anyhow::{ensure, Result};
+use searchi_core::SearchIndex;
+use searchi_media::{probe, FfmpegTools, MediaIdentity, MediaLimits};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(args.len() == 4, "Supply CORPUS FFMPEG FFPROBE OUTPUT");
+    let root = PathBuf::from(&args[0]).canonicalize()?;
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(lock["fixture_sha256"] == hash, "Frozen labels changed");
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    let tools = FfmpegTools::new(PathBuf::from(&args[1]), PathBuf::from(&args[2]))?;
+    let limits = MediaLimits::default();
+    let mut index = SearchIndex::open(":memory:")?;
+    index.index_folder(root.join("assets"))?;
+    let extraction_start = Instant::now();
+    for asset in fixture["assets"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|asset| asset["kind"] != "image")
+    {
+        let identity = MediaIdentity::read(
+            &root.join(asset["path"].as_str().unwrap()),
+            limits.max_file_bytes,
+        )?;
+        ensure!(
+            asset["sha256"] == identity.sha256,
+            "Frozen source asset changed"
+        );
+        let source = probe(&tools, &identity.path, &limits)?;
+        let tags = source
+            .tags
+            .iter()
+            .map(|(key, value)| format!("{key}: {value}"))
+            .collect::<Vec<_>>()
+            .join("\n");
+        index.set_media_metadata(
+            &identity.path.to_string_lossy(),
+            &tags,
+            source.duration_ms,
+            identity.bytes,
+            identity.modified_ns,
+        )?;
+    }
+    let extraction_ms = extraction_start.elapsed().as_secs_f64() * 1000.0;
+    let mut rows = Vec::new();
+    for query in fixture["queries"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|query| query.get("metadata").is_some())
+    {
+        let start = Instant::now();
+        let hits = index.search(query["text"].as_str().unwrap(), 20)?;
+        let query_ms = start.elapsed().as_secs_f64() * 1000.0;
+        let file_rankings: Vec<_> = hits.iter().map(|hit| {
+            let asset = fixture["assets"].as_array().unwrap().iter().find(|asset| PathBuf::from(asset["path"].as_str().unwrap()).file_name().unwrap().to_string_lossy()==hit.filename).unwrap();
+            json!({"candidate":{"asset":asset["id"],"modality":"metadata","match_kind":hit.match_kind},"score":hit.score})
+        }).collect();
+        rows.push(
+            json!({"query":query,"file_rankings":file_rankings,"segment_top5":[],"query_ms":query_ms}),
+        );
+    }
+    fs::write(
+        &args[3],
+        serde_json::to_vec_pretty(
+            &json!({"fixture_sha256":hash,"worker":null,"encoding_ms":extraction_ms,"encoding_failures":[],"rows":rows,"scope":"Actual FFprobe stream/format metadata + guarded Searchi core FTS; lexical identity queries, no embedding fallback; encoding_ms measures metadata extraction, not model inference"}),
+        )?,
+    )?;
+    Ok(())
+}

tools/media-eval/src/bin/pipeline.rs

@@ -0,0 +1,83 @@
+//! End-to-end corpus indexing and querying through actual Searchi APIs.
+use anyhow::{ensure, Result};
+use searchi_core::{SearchIndex, SearchOptions};
+use searchi_media::{FfmpegTools, MediaLimits};
+use searchi_service::HybridSearch;
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 6 || args.len() == 7,
+        "Supply CORPUS MODEL CPU_WORKER FFMPEG FFPROBE OUTPUT_JSON [AUDIO_WINDOW_SECONDS]"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU only"
+    );
+    let root = PathBuf::from(&args[0]).canonicalize()?;
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(lock["fixture_sha256"] == hash, "Frozen labels changed");
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    for asset in fixture["assets"].as_array().unwrap() {
+        let content = fs::read(root.join(asset["path"].as_str().unwrap()))?;
+        ensure!(
+            asset["sha256"] == format!("{:x}", Sha256::digest(content)),
+            "Asset changed"
+        );
+    }
+    let temporary = tempfile::tempdir()?;
+    let db = temporary.path().join("index.sqlite");
+    let mut lexical = SearchIndex::open(&db)?;
+    let filename_index = lexical.index_folder(root.join("assets"))?;
+    let mut hybrid = HybridSearch::open(&db, &args[1], &args[2], 1)?;
+    let tools = FfmpegTools::new(PathBuf::from(&args[3]), PathBuf::from(&args[4]))?;
+    let start = Instant::now();
+    let mut limits = MediaLimits::default();
+    if let Some(seconds) = args.get(6) {
+        let seconds: u32 = seconds.to_string_lossy().parse()?;
+        ensure!(
+            (5..=30).contains(&seconds),
+            "Audio window must be 5–30 seconds"
+        );
+        limits.audio_window_ms = seconds * 1000;
+    }
+    let index_report = hybrid.index_embeddings_with_media(&tools, &limits)?;
+    let encoding_ms = start.elapsed().as_secs_f64() * 1000.0;
+    println!("Actual pipeline indexed {} chunks", index_report.chunks);
+    hybrid.warmup()?;
+    let mut rows = Vec::new();
+    for query in fixture["queries"].as_array().unwrap() {
+        if query["filters"].get("time_range_s").is_some() {
+            rows.push(json!({"query":query,"unsupported":"Production query API has no temporal-range filter yet"}));
+            continue;
+        }
+        let mut options = SearchOptions::default();
+        if let Some(extension) = query["filters"]["extension"].as_str() {
+            options.extension = Some(extension.into());
+        }
+        let start = Instant::now();
+        let hits = hybrid.search(query["text"].as_str().unwrap(), 20, &options)?;
+        let query_ms = start.elapsed().as_secs_f64() * 1000.0;
+        let file_rankings: Vec<_> = hits.iter().map(|hit| {
+            let asset = fixture["assets"].as_array().unwrap().iter().find(|asset| PathBuf::from(asset["path"].as_str().unwrap()).file_name().unwrap().to_string_lossy() == hit.filename).unwrap();
+            let modality = if hit.match_kind.contains("audio") { "audio" } else if hit.match_kind == "media_metadata" { "metadata" } else { "image" };
+            let mut candidate = json!({"asset":asset["id"],"modality":modality,"match_kind":hit.match_kind,"source":hit.source});
+            if let Some(source) = &hit.source { if source.kind == "timestamp_ms" {
+                let stamp = f64::from(source.number)/1000.0;
+                candidate["start_s"] = json!(stamp);
+                candidate["end_s"] = json!((stamp + if modality == "audio" {f64::from(limits.audio_window_ms)/1000.0} else {2.0}).min(asset["duration_s"].as_f64().unwrap_or(stamp)));
+                candidate["interval_note"] = json!("End inferred from configured pipeline window policy, actual API exposes only start timestamp");
+            }}
+            json!({"candidate":candidate,"score":hit.score})
+        }).collect();
+        rows.push(json!({"query":query,"segment_top5":file_rankings.iter().take(5).cloned().collect::<Vec<_>>(),"file_rankings":file_rankings,"query_ms":query_ms}));
+    }
+    let report = json!({"fixture_sha256":hash,"worker":hybrid.worker_info(),"index_report":index_report,"filename_index":filename_index,"encoding_ms":encoding_ms,"audio_window_ms":limits.audio_window_ms,"audio_overlap_ms":limits.audio_overlap_ms,"encoding_failures":index_report.failures,"rows":rows,"scope":"Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API"});
+    fs::write(&args[5], serde_json::to_vec_pretty(&report)?)?;
+    Ok(())
+}

tools/media-eval/src/main.rs

@@ -0,0 +1,153 @@
+//! Actual CPU Google embeddings only; labels never enter document/image vectors.
+use anyhow::{ensure, Context, Result};
+use searchi_semantic::{Device, GemmaEmbedder, GEMMA_FINGERPRINT, GEMMA_MEDIA_FINGERPRINT};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf, time::Instant};
+
+fn main() -> Result<()> {
+    let args: Vec<_> = std::env::args_os().skip(1).collect();
+    ensure!(
+        args.len() == 4,
+        "Supply CORPUS_ROOT MODEL_DIR CPU_WORKER OUTPUT_JSON"
+    );
+    ensure!(
+        std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
+        "CPU-only benchmark requires SEARCHI_FORCE_CPU=1"
+    );
+    let root = PathBuf::from(&args[0]);
+    let bytes = fs::read(root.join("fixture.json"))?;
+    let hash = format!("{:x}", Sha256::digest(&bytes));
+    let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?;
+    ensure!(
+        lock["fixture_sha256"] == hash,
+        "Independent labels changed after freezing"
+    );
+    let fixture: Value = serde_json::from_slice(&bytes)?;
+    let candidates: Value =
+        serde_json::from_slice(&fs::read(root.join("derived/candidates.json"))?)?;
+    ensure!(
+        candidates["fixture_sha256"] == hash,
+        "Prepared media belongs to different labels"
+    );
+    let mut model = GemmaEmbedder::load_with_device(&args[1], &args[2], 1, Device::Cpu)?;
+    ensure!(
+        model.info.device.to_lowercase().contains("cpu"),
+        "Worker did not report CPU"
+    );
+    let start = Instant::now();
+    let mut vectors = Vec::new();
+    let mut encoded = Vec::new();
+    let mut encoding_failures = Vec::new();
+    for candidate in candidates["candidates"]
+        .as_array()
+        .context("candidates absent")?
+    {
+        let path = PathBuf::from(candidate["path"].as_str().context("path absent")?);
+        let result = if candidate["modality"] == "image" {
+            model
+                .embed_images_with_budget(
+                    &[&path],
+                    candidate["token_budget"].as_u64().unwrap() as u32,
+                )
+                .map(|mut rows| rows.remove(0))
+        } else {
+            let bytes = fs::read(&path)?;
+            ensure!(bytes.len() % 4 == 0, "PCM alignment invalid");
+            let samples: Vec<f32> = bytes
+                .as_chunks::<4>()
+                .0
+                .iter()
+                .map(|chunk| f32::from_le_bytes(*chunk))
+                .collect();
+            model.embed_audio_pcm(&samples)
+        };
+        match result {
+            Ok(vector) => {
+                vectors.push(vector);
+                encoded.push(candidate.clone());
+            }
+            Err(error) => {
+                encoding_failures.push(json!({"candidate": candidate, "error": error.to_string()}))
+            }
+        }
+        println!(
+            "Encoded {}/{} candidates",
+            encoded.len(),
+            candidates["candidates"].as_array().unwrap().len()
+        );
+    }
+    let index_ms = start.elapsed().as_secs_f64() * 1000.0;
+    let mut rows = Vec::new();
+    for query in fixture["queries"].as_array().context("queries absent")? {
+        if query.get("metadata").is_some() {
+            rows.push(json!({"query":query,"unsupported":"Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"}));
+            continue;
+        }
+        let start = Instant::now();
+        let embedding = model.embed_query(query["text"].as_str().context("query text absent")?)?;
+        let mut ranked = Vec::new();
+        for (candidate, vector) in encoded.iter().zip(&vectors) {
+            if let Some(modality) = query.get("required_modality") {
+                if candidate["modality"] != *modality {
+                    continue;
+                }
+            }
+            if let Some(filters) = query.get("filters") {
+                if let Some(asset) = filters.get("asset") {
+                    if candidate["asset"] != *asset {
+                        continue;
+                    }
+                }
+                if let Some(extension) = filters.get("extension").and_then(Value::as_str) {
+                    let asset = fixture["assets"]
+                        .as_array()
+                        .unwrap()
+                        .iter()
+                        .find(|asset| asset["id"] == candidate["asset"])
+                        .unwrap();
+                    if !asset["path"]
+                        .as_str()
+                        .unwrap()
+                        .ends_with(&format!(".{extension}"))
+                    {
+                        continue;
+                    }
+                }
+                if let Some(range) = filters.get("time_range_s") {
+                    let stamp = candidate["start_s"].as_f64().unwrap_or(-1.0);
+                    if stamp < range[0].as_f64().unwrap() || stamp > range[1].as_f64().unwrap() {
+                        continue;
+                    }
+                }
+            }
+            ensure!(
+                embedding.len() == vector.len(),
+                "Mixed embedding dimensions"
+            );
+            let score: f64 = embedding
+                .iter()
+                .zip(vector)
+                .map(|(a, b)| f64::from(*a) * f64::from(*b))
+                .sum();
+            ranked.push(json!({"candidate":candidate,"score":score}));
+        }
+        ranked.sort_by(|a, b| {
+            b["score"]
+                .as_f64()
+                .unwrap()
+                .total_cmp(&a["score"].as_f64().unwrap())
+        });
+        let mut seen = std::collections::HashSet::new();
+        let file_rankings: Vec<_> = ranked
+            .iter()
+            .filter(|row| seen.insert(row["candidate"]["asset"].as_str().unwrap()))
+            .take(20)
+            .cloned()
+            .collect();
+        rows.push(json!({"query":query,"file_rankings":file_rankings,"segment_top5":ranked.into_iter().take(5).collect::<Vec<_>>(),"query_ms":start.elapsed().as_secs_f64()*1000.0}));
+    }
+    let report = json!({"fixture_sha256":hash,"worker":model.info,"text_fingerprint":GEMMA_FINGERPRINT,"media_fingerprint":GEMMA_MEDIA_FINGERPRINT,"policy":candidates["policy"],"ffmpeg_version":candidates["ffmpeg_version"],"encoding_ms":index_ms,"encoded_candidates":encoded.len(),"encoding_failures":encoding_failures,"rows":rows,"scope":"Direct real-model dense reference baseline; candidate filters here are benchmark reference constraints, not production-filter certification; no rejection threshold is tuned"});
+    fs::write(&args[3], serde_json::to_vec_pretty(&report)?)?;
+    Ok(())
+}