Validate shared-engine lexical performance harness on Windows
This commit is contained in:
karl committed
1 parent
fb8b45dcc9
commit
4e277a107f
6 files changed
+1534
No files matched your search
@@ -0,0 +1,250 @@
|
||||
//! Synthetic index performance only; no physical corpus, extraction or inference.
|
||||
use anyhow::{bail, Context, Result};
|
||||
use rusqlite::{params, Connection};
|
||||
use searchi_core::{SearchIndex, SearchOptions};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{env, fs, io::Read, path::Path, time::Instant};
|
||||
|
||||
const SEED: u64 = 20261013;
|
||||
const LIMIT: usize = 20;
|
||||
|
||||
fn hash(path: &Path) -> Result<String> {
|
||||
let mut file = fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0u8; 65536];
|
||||
loop {
|
||||
let count = file.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..count]);
|
||||
}
|
||||
Ok(format!("{:x}", digest.finalize()))
|
||||
}
|
||||
|
||||
fn seed(db: &Path, root: &Path, files: usize, docs: usize) -> Result<()> {
|
||||
// Let the actual engine create/migrate its schema and all FTS triggers.
|
||||
drop(SearchIndex::open(db)?);
|
||||
let mut connection = Connection::open(db)?;
|
||||
for start in (0..files).step_by(1000) {
|
||||
let transaction = connection.transaction()?;
|
||||
{
|
||||
let mut insert = transaction.prepare("INSERT INTO files(path,root,filename,content,size,mtime,extension) VALUES(?1,?2,?3,?4,?5,?6,?7)")?;
|
||||
let mut passage = transaction.prepare(
|
||||
"INSERT INTO passages(text,file_path,kind,number) VALUES(?1,?2,'line',1)",
|
||||
)?;
|
||||
for id in start..(start + 1000).min(files) {
|
||||
let extension = if id.is_multiple_of(2) { "txt" } else { "rs" };
|
||||
let filename = format!("record_{id:07}.{extension}");
|
||||
let path = root
|
||||
.join(if id % 3 == 0 {
|
||||
"chosen"
|
||||
} else {
|
||||
"chosen-other"
|
||||
})
|
||||
.join(&filename);
|
||||
let path = path.to_string_lossy();
|
||||
let content = if id < docs {
|
||||
format!("Synthetic document uniquemarker{id:07} cancellation renewal notice category{} seed{SEED}", id % 97)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
insert.execute(params![
|
||||
path,
|
||||
root.to_string_lossy(),
|
||||
filename,
|
||||
content,
|
||||
(id % 8192 + 100) as i64,
|
||||
(1700000000000i64 + id as i64) * 1_000_000,
|
||||
extension
|
||||
])?;
|
||||
if !content.is_empty() {
|
||||
passage.execute(params![content, path])?;
|
||||
}
|
||||
}
|
||||
}
|
||||
transaction.commit()?;
|
||||
}
|
||||
for table in ["search", "names", "content_search", "passage_search"] {
|
||||
// content_search intentionally omits empty files; external-content
|
||||
// comparison would incorrectly require those omitted row IDs.
|
||||
let rank = if table == "content_search" { 0 } else { 1 };
|
||||
connection
|
||||
.execute_batch(&format!(
|
||||
"INSERT INTO {table}({table},rank) VALUES('integrity-check',{rank});"
|
||||
))
|
||||
.with_context(|| format!("FTS integrity check for {table}"))?;
|
||||
}
|
||||
let integrity: String = connection.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
if integrity != "ok" {
|
||||
bail!("SQLite integrity check: {integrity}");
|
||||
}
|
||||
let counts: (usize, usize, usize) = connection.query_row(
|
||||
"SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM files WHERE content!=''),(SELECT count(*) FROM passages)",
|
||||
[], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
||||
)?;
|
||||
if counts != (files, docs, docs) {
|
||||
bail!("seed row counts differ: {counts:?}");
|
||||
}
|
||||
connection.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run(index: &SearchIndex, case: &str, id: usize, root: &Path, debug: bool) -> Result<Value> {
|
||||
let mut options = SearchOptions::default();
|
||||
let query = match case {
|
||||
"exact" => format!(
|
||||
"record_{id:07}.{}",
|
||||
if id.is_multiple_of(2) { "txt" } else { "rs" }
|
||||
),
|
||||
"keyword_unique" => format!("uniquemarker{id:07}"),
|
||||
"common_keyword" => "cancellation".into(),
|
||||
"common_name" => "record_".into(),
|
||||
"strict_filters" => {
|
||||
options.folder = Some(root.join("chosen"));
|
||||
options.extension = Some("rs".into());
|
||||
options.code_only = true;
|
||||
options.min_size = Some(110);
|
||||
options.max_size = Some(119);
|
||||
options.modified_after_unix_ms = Some(1700000000010);
|
||||
options.modified_before_unix_ms = Some(1700000000019);
|
||||
"cancellation".into()
|
||||
}
|
||||
"absent" => "nonexistentzzxyartifact".into(),
|
||||
_ => bail!("unknown case"),
|
||||
};
|
||||
let start = Instant::now();
|
||||
let response = index.search_with_options_measured(&query, LIMIT, &options, debug)?;
|
||||
let wall_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||
if case == "absent" {
|
||||
if !response.results.is_empty() {
|
||||
bail!("absent query returned results");
|
||||
}
|
||||
} else {
|
||||
let first = response
|
||||
.results
|
||||
.first()
|
||||
.with_context(|| format!("positive {case} query returned nothing: {query}"))?;
|
||||
if case == "exact" && (first.filename != query || first.match_kind != "exact_filename") {
|
||||
bail!("exact filename priority failed");
|
||||
}
|
||||
if case == "keyword_unique" && !first.filename.starts_with(&format!("record_{id:07}.")) {
|
||||
bail!("unique keyword identity failed");
|
||||
}
|
||||
if case == "strict_filters"
|
||||
&& (response.results.len() != 1 || first.filename != "record_0000015.rs")
|
||||
{
|
||||
bail!("strict filter expected exactly the seeded eligible record 15");
|
||||
}
|
||||
for result in &response.results {
|
||||
if case == "strict_filters"
|
||||
&& (!Path::new(&result.path).starts_with(root.join("chosen"))
|
||||
|| !result.filename.ends_with(".rs"))
|
||||
{
|
||||
bail!("strict folder/code/extension filter failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug != response.metrics.trace.is_some() {
|
||||
bail!("trace mode mismatch");
|
||||
}
|
||||
Ok(json!({"wall_ms":wall_ms,"metrics":response.metrics}))
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<String> = env::args().skip(1).collect();
|
||||
if args.len() != 4 {
|
||||
bail!("usage: searchi-scale-eval FILES DOCUMENTS SAMPLES REPORT.json");
|
||||
}
|
||||
let files: usize = args[0].parse()?;
|
||||
let docs: usize = args[1].parse()?;
|
||||
let samples: usize = args[2].parse()?;
|
||||
if !(100..=1000000).contains(&files)
|
||||
|| !(20..=100000).contains(&docs)
|
||||
|| docs > files
|
||||
|| !(20..=1000).contains(&samples)
|
||||
{
|
||||
bail!("require 100..1M files, 20..100K docs <= files, 20..1000 samples");
|
||||
}
|
||||
// TempDir owns exactly this directory. No caller directory is recursively removed.
|
||||
let owned = tempfile::Builder::new()
|
||||
.prefix("searchi-scale-")
|
||||
.tempdir()?;
|
||||
let root = owned.path().join("corpus");
|
||||
for folder in ["chosen", "chosen-other"] {
|
||||
fs::create_dir_all(root.join(folder))?;
|
||||
}
|
||||
let root = fs::canonicalize(root)?;
|
||||
let db = owned.path().join("index.sqlite");
|
||||
let seeding = Instant::now();
|
||||
seed(&db, &root, files, docs)?;
|
||||
let seed_ms = seeding.elapsed().as_secs_f64() * 1000.0;
|
||||
let first_open = Instant::now();
|
||||
let mut index = SearchIndex::open(&db)?;
|
||||
let open_ms = first_open.elapsed().as_secs_f64() * 1000.0;
|
||||
let unprepared_first = run(&index, "exact", files / 2, &root, false)?;
|
||||
let prepare = Instant::now();
|
||||
let cached_frequencies = index.prewarm()?;
|
||||
let prepare_ms = prepare.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut cases = Vec::new();
|
||||
for case in [
|
||||
"exact",
|
||||
"keyword_unique",
|
||||
"common_keyword",
|
||||
"common_name",
|
||||
"strict_filters",
|
||||
"absent",
|
||||
] {
|
||||
let first = run(&index, case, docs / 2 + 1, &root, false)?;
|
||||
let mut basic = Vec::with_capacity(samples);
|
||||
let mut debug = Vec::with_capacity(samples);
|
||||
for sample in 0..samples {
|
||||
// Paired AB/BA to avoid systematically favoring one trace mode.
|
||||
let id = (sample * 7919 + SEED as usize) % docs;
|
||||
for detailed in if sample.is_multiple_of(2) {
|
||||
[false, true]
|
||||
} else {
|
||||
[true, false]
|
||||
} {
|
||||
let value = run(&index, case, id, &root, detailed)?;
|
||||
if detailed {
|
||||
debug.push(value);
|
||||
} else {
|
||||
basic.push(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
cases.push(json!({"case":case,"first_after_prewarm":first,"basic":basic,"debug":debug}));
|
||||
}
|
||||
drop(index);
|
||||
let restart = Instant::now();
|
||||
let reopened = SearchIndex::open(&db)?;
|
||||
let reopened_open_ms = restart.elapsed().as_secs_f64() * 1000.0;
|
||||
let reopened_first = run(&reopened, "keyword_unique", docs - 1, &root, false)?;
|
||||
drop(reopened);
|
||||
let connection = Connection::open(&db)?;
|
||||
let schema: i64 = connection.query_row("PRAGMA user_version", [], |r| r.get(0))?;
|
||||
let plan: Vec<String> = connection
|
||||
.prepare(
|
||||
"EXPLAIN QUERY PLAN SELECT id FROM files WHERE lower(filename)='record_0000001.rs'",
|
||||
)?
|
||||
.query_map([], |r| r.get(3))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
connection.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")?;
|
||||
drop(connection);
|
||||
let report = json!({"kind":"synthetic_lexical_index_only","seed":SEED,"files":files,"documents":docs,"schema":schema,
|
||||
"os":env::consts::OS,"arch":env::consts::ARCH,"logical_cpus":std::thread::available_parallelism()?.get(),
|
||||
"executable_sha256":hash(&env::current_exe()?)?,"seed_ms":seed_ms,"database_bytes":fs::metadata(&db)?.len(),
|
||||
"database_sha256":hash(&db)?,"first_reader_open_ms":open_ms,"unprepared_first_input":unprepared_first,
|
||||
"prewarm_ms":prepare_ms,"cached_frequencies":cached_frequencies,"reopened_open_ms":reopened_open_ms,
|
||||
"reopened_first_input":reopened_first,"exact_lookup_plan":plan,"cases":cases,
|
||||
"cold_storage_tested":false,"physical_files_created":0,"inference":false});
|
||||
// create_new prevents accidental replacement of previous evidence.
|
||||
let output = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&args[3])?;
|
||||
serde_json::to_writer_pretty(output, &report)?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user