feat: add multimodal search and granular performance diagnostics

crates/searchi-core/examples/trace_overhead.rs

@@ -0,0 +1,80 @@
+//! Small prepared SQLite instrumentation diagnostic; not a production benchmark.
+use anyhow::{ensure, Result};
+use searchi_core::SearchIndex;
+use std::{fs, path::Path, time::Instant};
+
+fn distribution(samples: &[f64]) -> serde_json::Value {
+    let mut sorted = samples.to_vec();
+    sorted.sort_by(f64::total_cmp);
+    let percentile = |fraction: f64| {
+        sorted[((sorted.len() as f64 * fraction).ceil() as usize).saturating_sub(1)]
+    };
+    serde_json::json!({"samples":sorted.len(),"p50_ms":percentile(0.50),"p95_ms":percentile(0.95),"mean_ms":sorted.iter().sum::<f64>()/sorted.len() as f64})
+}
+fn main() -> Result<()> {
+    let fixture = tempfile::tempdir()?;
+    let root = fixture.path().join("files");
+    fs::create_dir(&root)?;
+    for index in 0..1000 {
+        fs::write(
+            root.join(format!("record-{index:04}.txt")),
+            format!(
+                "Supplier agreement {index}. Cancellation needs written notice. Group {}.",
+                index % 31
+            ),
+        )?;
+    }
+    let mut index = SearchIndex::open(fixture.path().join("index.sqlite"))?;
+    let report = index.index_folder(&root)?;
+    index.prewarm()?;
+    ensure!(report.indexed == 1000, "Unexpected fixture indexing count");
+    let queries: Vec<String> = (0..24)
+        .map(|index| format!("record-{:04}.txt", index * 37))
+        .collect();
+    for query in &queries {
+        index.search_measured(query, 10, false)?;
+        index.search_measured(query, 10, true)?;
+    }
+    let mut basic = Vec::new();
+    let mut debug = Vec::new();
+    let mut trace_spans = Vec::new();
+    // Alternate order to reduce monotonic cache/CPU-order effects.
+    for repetition in 0..20 {
+        for query in &queries {
+            for detailed in if repetition % 2 == 0 {
+                [false, true]
+            } else {
+                [true, false]
+            } {
+                let started = Instant::now();
+                let response = index.search_measured(query, 10, detailed)?;
+                let elapsed = started.elapsed().as_secs_f64() * 1000.0;
+                ensure!(
+                    response
+                        .results
+                        .first()
+                        .is_some_and(|result| &result.filename == query),
+                    "Fixture retrieval changed"
+                );
+                if detailed {
+                    trace_spans.push(response.metrics.trace.as_ref().unwrap().len());
+                    debug.push(elapsed);
+                } else {
+                    ensure!(
+                        response.metrics.trace.is_none(),
+                        "Basic unexpectedly stored traces"
+                    );
+                    basic.push(elapsed);
+                }
+            }
+        }
+    }
+    let output = serde_json::json!({"kind":"prepared_sqlite_trace_overhead","production_claim":false,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"files":1000,"queries":24,"repetitions":20,"measurement":"outer Rust call including metrics assembly; excludes fixture setup/JSON serialization","basic":distribution(&basic),"debug":distribution(&debug),"basic_samples_ms":basic,"debug_samples_ms":debug,"debug_span_count_min":trace_spans.iter().min(),"debug_span_count_max":trace_spans.iter().max(),"limits":"tiny warm lexical corpus; harness serial/no model/media/GPU; other VM work including native photo inference may contend; not isolated production performance"});
+    let serialized = serde_json::to_string_pretty(&output)?;
+    if let Some(path) = std::env::args_os().nth(1) {
+        fs::write(Path::new(&path), serialized)?;
+    } else {
+        println!("{serialized}");
+    }
+    Ok(())
+}