feat: add multimodal search and granular performance diagnostics

crates/searchi-semantic/tests/fixtures/README.md

@@ -0,0 +1,14 @@
+# Pinned Google media reference fixtures
+
+The `digup-photo.jpg`, `digup-screen.png`, `digup-speech.wav`, and
+`digup-media-reference.json` files are copied unchanged from
+`ARahim3/DigUp`, commit `6faa29e4c315a3a591de1b41bd78751bdf39588d`,
+`Tests/LlamaRuntimeTests/Reference/`. They and the existing text reference
+fixture are retained under the adjacent `LICENSE-DigUp.txt` MIT notice.
+
+The source test uses image budgets 280 (screen), 140 (photo), 70 (frame),
+and a mono 16 kHz speech recording. The native CPU test compares actual
+768-dimensional Google vectors with the pinned llama.cpp b11461 reference
+at cosine >=0.999. This is runtime parity evidence, not a production media
+relevance corpus. Independent images, audio, and video require separate
+quality evaluation.

crates/searchi-semantic/tests/fixtures/digup-media-reference.json

@@ -0,0 +1 @@
+{"made_with":"llama.cpp b11461","vector_version":1,"vectors":{"document":[0.0023304052,-0.021797102,-0.025986522,-0.025894934,0.0046086097,-0.029744277,-0.04186634,0.03234369,0.015701862,-0.13444579,-0.049679857,-0.067124754,-0.008888296,0.049783457,0.04075608,-0.0027419934,0.0494676,0.0092225745,-0.003042357,0.021539804,-0.066572726,-0.04184304,-0.008146685,-0.015800655,0.0028940293,0.026850378,0.023585942,0.012522315,-0.018817414,0.19809185,-0.0541878,-0.033899978,0.06363426,-0.012184333,0.026218394,-0.030783277,-0.016514625,0.0010143976,0.016249185,0.00673781,0.07569905,-0.032087695,0.01618515,0.008316216,0.009405525,0.035788074,0.011760172,-0.0018986556,0.03811712,0.04686964,-0.0019338331,-0.06858261,-0.020950682,0.0061196685,-0.012488069,0.055386145,0.00828158,-0.028973866,0.029866058,0.019298207,0.0064102146,0.03210796,0.031232368,-0.024290783,-0.009193791,-0.00788917,0.07007842,-0.036232397,0.048318822,0.03329114,-0.021271994,-0.01810213,0.056723867,0.026635557,-0.0066668643,-0.03339305,-0.09275154,-0.005457068,-0.037602138,0.06852603,-0.086277,0.0108406255,-0.022301262,0.054799795,0.048695795,-0.0094250385,0.04326083,-0.05584021,0.009427591,0.018164966,0.0077456376,0.012890579,0.011140378,0.009403773,-0.013322473,0.014014791,0.0045265807,0.0026267597,-0.02777846,0.03337617,-0.014311906,-0.0027436074,0.0016559366,0.04267254,-0.0074306116,0.010555651,0.00801582,0.019190526,0.026427642,-0.02305468,0.017346293,-0.254231,-0.037654083,0.004254816,-0.029559055,0.30139065,-0.0017701226,0.012945749,0.045082387,-0.005745192,0.023981802,0.11611545,0.02849787,0.0050043385,0.019032571,0.01543062,0.013486391,-0.0073420904,-0.0120351305,0.027507395,0.017718071,0.0048503615,0.0070377868,-0.00946547,0.015142535,0.013139711,0.019906517,-0.038481727,0.060138095,0.0027306078,-0.010124241,0.020947324,-0.005082965,0.009129657,-0.06960446,0.04814667,0.012469531,-0.017168682,-0.007835558,-0.018339474,0.04499317,-0.048769984,-0.021295255,0.0008460758,0.042394932,0.02629723,0.015862137,-0.027827857,0.02877754,0.044078708,0.02864664,0.015502866,-0.011931367,0.006051371,-0.0676744,-0.032051444,-0.0051915697,0.022663914,0.010766299,-0.045957707,-0.071434505,-0.0071970616,-0.00080499344,-0.019176258,0.039329063,0.07615527,-0.02934876,0.03178097,0.030727895,-0.0014563488,0.033275653,-0.016663045,0.016650142,0.03149862,-0.042355653,-0.01447139,-0.031222051,-0.0371221,-0.071084455,0.02895739,0.012906462,-0.034588896,0.005292062,0.04567936,0.057948757,0.016161263,0.026154758,0.037441123,0.020264363,0.0021852585,0.010503882,0.02564951,-0.0034963347,-0.002975977,-0.05896342,0.090469755,-0.02141068,-0.018266847,0.03979448,0.006404804,0.048994806,-0.0013431136,0.003767527,0.009192836,0.03172842,0.0018301627,-0.032883424,0.070755646,-0.008929331,0.007731602,0.039488316,-0.0026821592,0.047416102,0.017939921,-0.046493225,-0.003571527,0.017453436,0.027679116,-0.045083456,-0.0018627502,-0.0058596916,-0.016741525,-0.041559335,-0.07398639,0.035852198,-0.01250213,-0.027620463,-0.022970477,-0.007722595,0.006369175,-0.027909646,0.05087572,0.04319605,0.0249135,-0.045503743,-0.050388955,-0.07399011,-0.031614304,-0.009977491,-0.011919013,0.017587284,0.046735547,0.051120915,-0.01913088,-0.0040965537,-0.030126736,0.022246089,-0.024694057,-0.0011967963,-0.0076620826,-0.0001045683,0.025630128,0.0023922755,-0.016251149,0.029261485,0.026775412,-0.0011348778,-0.023079926,0.0046075485,0.013615946,-0.022271913,0.0063756416,-0.011097959,-0.057059642,0.017879428,0.039326653,-0.010385263,-0.024296982,-0.0056849266,-0.021441974,0.023893535,-0.004312103,-0.0029023795,0.011346968,-0.05022057,-0.00962676,-0.030292084,0.0022458949,-0.0012382208,0.014016518,0.0130033605,-0.0075721424,0.022480633,0.0378293,-0.0060748337,0.020296352,-0.038013346,0.02737668,0.051704004,0.029559156,-0.008555367,-0.040540576,0.0028047708,-0.01888827,-0.017475538,0.022049233,-0.018655647,0.00035712068,0.0007242559,-0.031585597,0.002809768,-0.010847336,0.019464653,-0.038227003,0.004094894,-0.0045381063,-0.032818098,-0.025449943,-0.0048597767,-0.052715924,0.024247607,-0.025617195,-0.028776655,-0.01491966,0.027141426,0.022653097,0.01347185,0.0010195217,0.0077305734,0.006165836,-0.009121481,-0.011312435,0.0010023145,0.033321623,0.0020216978,-0.04601626,-0.020557044,-0.023531808,0.013099387,0.049464565,-0.0083905,-0.12218178,-0.0090951715,-0.037435923,-0.043180246,0.008593723,0.003133877,0.03239143,-0.02740052,0.028175933,0.00713765,0.010822759,-0.011402951,-0.042760924,-0.03611006,0.011457286,0.060568936,0.0027335465,-0.024305783,-0.004420147,0.0068441983,0.002837347,-0.02218109,-0.022486687,-0.01941801,0.01667096,-0.03958557,-0.0077348636,-0.0012779543,-0.009598079,-0.008962751,0.033793483,0.018625712,-0.006633453,0.026242023,0.021685809,0.013799533,-0.0060389466,0.0020937775,0.02297482,-0.02592566,0.0011351658,0.00793861,-0.01809629,0.050504446,-0.026299631,-0.028017156,0.006029957,-0.02648004,0.03030571,0.016809817,0.29351303,-0.028621448,0.01694766,0.021401705,0.012320341,0.0015229117,-0.Line truncated

crates/searchi-semantic/tests/gemma_cpu.rs

@@ -1,6 +1,49 @@
 use anyhow::Result;
 use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
 
+#[test]
+#[ignore = "Requires explicitly provisioned Google model and CPU worker"]
+fn google_cpu_debug_query_reports_actual_worker_stages_without_changing_vectors() -> Result<()> {
+    let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set SEARCHI_GEMMA_MODEL_DIR");
+    let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set SEARCHI_GEMMA_WORKER");
+    let mut embedder = GemmaEmbedder::load_with_device(model, worker, 1, Device::Cpu)?;
+    let query = "supplier contract cancellation policy";
+    let debug = embedder.embed_query_debug(query)?;
+    let diagnostics = embedder.query_diagnostics();
+    for name in [
+        "request_encode_us",
+        "request_write_us",
+        "worker_wait_and_response_decode_us",
+        "response_validate_us",
+        "worker_tokenization_us",
+        "worker_batch_prepare_us",
+        "worker_inference_us",
+        "worker_pool_normalize_us",
+        "worker_token_count",
+    ] {
+        assert!(
+            diagnostics.contains_key(name),
+            "Missing actual stage {name}: {diagnostics:?}"
+        );
+    }
+    assert!(diagnostics["worker_inference_us"] > 0);
+    assert!(diagnostics["worker_token_count"] > 0);
+    assert!(!serde_json::to_string(&diagnostics)?.contains(query));
+    let normal = embedder.embed_query(query)?;
+    assert!(embedder.query_diagnostics().is_empty());
+    let cosine: f64 = debug
+        .iter()
+        .zip(&normal)
+        .map(|(a, b)| f64::from(*a) * f64::from(*b))
+        .sum();
+    assert!(
+        cosine >= 0.99999,
+        "Debug tracing changed embeddings: {cosine}"
+    );
+    println!("CPU query diagnostics: {diagnostics:?}; debug/normal cosine {cosine:.8}");
+    Ok(())
+}
+
 #[test]
 #[ignore = "Requires explicitly provisioned Google model and CPU worker"]
 fn google_cpu_real_embeddings_and_verified_tail_windows() -> Result<()> {

crates/searchi-semantic/tests/gemma_media_cpu.rs

@@ -0,0 +1,57 @@
+use anyhow::Result;
+use searchi_semantic::{Device, GemmaEmbedder};
+use std::path::Path;
+
+#[test]
+#[ignore = "Requires explicitly provisioned Google media model and CPU worker"]
+fn google_cpu_pixels_match_digup_reference() -> Result<()> {
+    let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set model path");
+    let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker path");
+    let mut embedder = GemmaEmbedder::load_with_device(model, worker, 1, Device::Cpu)?;
+    let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
+    let reference: serde_json::Value =
+        serde_json::from_str(include_str!("fixtures/digup-media-reference.json"))?;
+    for (name, file, budget) in [
+        ("screen-280", "digup-screen.png", 280),
+        ("photo-140", "digup-photo.jpg", 140),
+        ("frame-70", "digup-photo.jpg", 70),
+    ] {
+        let path = fixtures.join(file);
+        let vector = embedder
+            .embed_images_with_budget(&[path.as_path()], budget)?
+            .remove(0);
+        let cosine: f64 = vector
+            .iter()
+            .zip(reference["vectors"][name].as_array().unwrap())
+            .map(|(a, b)| f64::from(*a) * b.as_f64().unwrap())
+            .sum();
+        println!("Google image {name} reference cosine={cosine:.8}");
+        assert!(cosine >= 0.999, "Image parity below0.999: {name} {cosine}");
+    }
+    assert_eq!(embedder.info.device, "cpu");
+    let wav = include_bytes!("fixtures/digup-speech.wav");
+    let mut offset = 12;
+    let mut samples = Vec::new();
+    while offset + 8 <= wav.len() {
+        let length = u32::from_le_bytes(wav[offset + 4..offset + 8].try_into()?) as usize;
+        if &wav[offset..offset + 4] == b"data" {
+            for pair in wav[offset + 8..offset + 8 + length].as_chunks::<2>().0 {
+                samples.push(f32::from(i16::from_le_bytes(*pair)) / 32768.0);
+            }
+            break;
+        }
+        offset += 8 + length + (length % 2);
+    }
+    let vector = embedder.embed_audio_pcm(&samples)?;
+    let cosine: f64 = vector
+        .iter()
+        .zip(reference["vectors"]["speech"].as_array().unwrap())
+        .map(|(a, b)| f64::from(*a) * b.as_f64().unwrap())
+        .sum();
+    println!("Google audio reference cosine={cosine:.8}");
+    assert!(
+        cosine >= 0.999,
+        "Audio reference parity below0.999: {cosine}"
+    );
+    Ok(())
+}