//! CPU-only synthetic retrieval benchmark. This does not measure filesystem //! scanning, extraction, embedding inference, or desktop rendering performance. use anyhow::{bail, Context, Result}; use rusqlite::{params, Connection}; use searchi_core::SearchIndex; use serde::Serialize; use std::{env, fs, path::PathBuf, time::Instant}; #[derive(Serialize)] struct Measurements { query: String, samples: usize, p50_ms: f64, first_query_ms: f64, p95_ms: f64, target_p95_ms: f64, target_met: bool, } #[derive(Serialize)] struct Report { fixture: &'static str, files: usize, documents: usize, seed_seconds: f64, migration_seconds: f64, fixture_reused: bool, database_bytes: u64, database_path: String, architecture: &'static str, os: &'static str, cpu_only: bool, queries: Vec, } fn main() -> Result<()> { let arguments = env::args().skip(1).collect::>(); let mut files = 10_000usize; let mut documents = 1_000usize; let mut samples = 20usize; let mut database = PathBuf::from("target/benchmarks/synthetic.db"); let mut reuse = false; for pair in arguments.chunks(2) { if pair.len() != 2 { bail!("usage: performance --files N --documents N --samples N --db NEW_PATH"); } match pair[0].as_str() { "--files" => files = pair[1].parse()?, "--documents" => documents = pair[1].parse()?, "--samples" => samples = pair[1].parse()?, "--db" => database = PathBuf::from(&pair[1]), "--reuse" => reuse = pair[1].parse()?, option => bail!("unknown argument {option}"), } } if files == 0 || files > 2_000_000 || documents == 0 || documents > files || !(5..=1000).contains(&samples) { bail!("files must be1..2000000, documents1..files, samples5..1000"); } if database.exists() && !reuse { bail!( "benchmark database must be a new path: {}", database.display() ); } if reuse && !database.exists() { bail!("reuse requires an existing synthetic fixture"); } if let Some(parent) = database.parent() { fs::create_dir_all(parent)?; } // Seed through the actual versioned schema and its normal indexing triggers. if reuse { let connection = Connection::open(&database)?; let (actual_files, actual_documents, foreign_roots): (usize,usize,usize) = connection.query_row( "SELECT count(*),count(NULLIF(content,'')),sum(root!='synthetic://benchmark') FROM files",[], |row|Ok((row.get(0)?,row.get(1)?,row.get(2)?)))?; if (actual_files, actual_documents, foreign_roots) != (files, documents, 0) { bail!("existing fixture does not match configured synthetic corpus"); } } let migration_started = Instant::now(); let index = SearchIndex::open(&database)?; drop(index); let migration_seconds = migration_started.elapsed().as_secs_f64(); let start = Instant::now(); let mut connection = Connection::open(&database)?; connection.execute_batch("PRAGMA synchronous=NORMAL;")?; if !reuse { for batch_start in (0..files).step_by(5000) { let transaction = connection.transaction()?; { let mut statement = transaction.prepare( "INSERT INTO files(path,root,filename,content,size,mtime,extension) VALUES (?1,'synthetic://benchmark',?2,?3,?4,0,'txt')", )?; let mut passage = transaction.prepare( "INSERT INTO passages(text,file_path,kind,number) VALUES (?1,?2,'paragraph',1)", )?; for number in batch_start..(batch_start + 5000).min(files) { let filename = format!("invoice_{number:07}.txt"); let path = format!( "synthetic://benchmark/folder_{:04}/{filename}", number % 1000 ); let content = if number < documents { format!("Supplier termination cancellation clause budget analysis marker{number:07}") } else { String::new() }; statement.execute(params![path, filename, content, content.len() as i64])?; if number < documents { passage.execute(params![content, path])?; } } } transaction.commit()?; if batch_start % 100_000 == 0 { eprintln!( "seeded {} / {files} synthetic file records", (batch_start + 5000).min(files) ); } } } connection.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")?; drop(connection); let seed_seconds = if reuse { 0.0 } else { start.elapsed().as_secs_f64() }; let database_bytes = fs::metadata(&database)?.len(); let index = SearchIndex::open(&database)?; let exact = format!("invoice_{:07}.txt", files - 1); let unique_content = format!("marker{:07}", documents - 1); let cases = [ (exact, 50.0), ("invoice_".into(), 50.0), (unique_content, 150.0), ("cancellation".into(), 150.0), ]; let mut queries = Vec::new(); for (query, target) in cases { let first_started = Instant::now(); index.search(&query, 20)?; let first_query_ms = first_started.elapsed().as_secs_f64() * 1000.0; for _ in 0..2 { index.search(&query, 20)?; } let mut durations = Vec::with_capacity(samples); for _ in 0..samples { let started = Instant::now(); let results = index .search(&query, 20) .with_context(|| format!("benchmark query {query}"))?; if results.is_empty() { bail!("synthetic query {query} unexpectedly returned no results"); } std::hint::black_box(results); durations.push(started.elapsed().as_secs_f64() * 1000.0); } durations.sort_by(f64::total_cmp); let percentile = |fraction: f64| { durations[((durations.len() as f64 * fraction).ceil() as usize).saturating_sub(1)] }; let p95_ms = percentile(0.95); eprintln!("query {query:?}: p95 {p95_ms:.3} ms; target < {target:.0} ms"); queries.push(Measurements { query, samples, p50_ms: percentile(0.5), first_query_ms, p95_ms, target_p95_ms: target, target_met: p95_ms < target, }); } let report = Report { fixture: "synthetic SQLite metadata and text; no filesystem extraction or semantic inference", files, documents, seed_seconds, migration_seconds, fixture_reused: reuse, database_bytes, database_path: database.display().to_string(), architecture: env::consts::ARCH, os: env::consts::OS, cpu_only: true, queries, }; println!("{}", serde_json::to_string_pretty(&report)?); Ok(()) }