Validate shared-engine lexical performance harness on Windows
This commit is contained in:
karl committed
1 parent
fb8b45dcc9
commit
4e277a107f
6 files changed
+1534
No files matched your search
Generated
+1072
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "searchi-scale-eval"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
publish = false
|
||||
|
||||
# Diagnostic only: intentionally outside the product workspace.
|
||||
[workspace]
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
searchi-core = { path = "../../crates/searchi-core" }
|
||||
serde_json = "1"
|
||||
sha2 = "0.10"
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,250 @@
|
||||
//! Synthetic index performance only; no physical corpus, extraction or inference.
|
||||
use anyhow::{bail, Context, Result};
|
||||
use rusqlite::{params, Connection};
|
||||
use searchi_core::{SearchIndex, SearchOptions};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{env, fs, io::Read, path::Path, time::Instant};
|
||||
|
||||
const SEED: u64 = 20261013;
|
||||
const LIMIT: usize = 20;
|
||||
|
||||
fn hash(path: &Path) -> Result<String> {
|
||||
let mut file = fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0u8; 65536];
|
||||
loop {
|
||||
let count = file.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..count]);
|
||||
}
|
||||
Ok(format!("{:x}", digest.finalize()))
|
||||
}
|
||||
|
||||
fn seed(db: &Path, root: &Path, files: usize, docs: usize) -> Result<()> {
|
||||
// Let the actual engine create/migrate its schema and all FTS triggers.
|
||||
drop(SearchIndex::open(db)?);
|
||||
let mut connection = Connection::open(db)?;
|
||||
for start in (0..files).step_by(1000) {
|
||||
let transaction = connection.transaction()?;
|
||||
{
|
||||
let mut insert = transaction.prepare("INSERT INTO files(path,root,filename,content,size,mtime,extension) VALUES(?1,?2,?3,?4,?5,?6,?7)")?;
|
||||
let mut passage = transaction.prepare(
|
||||
"INSERT INTO passages(text,file_path,kind,number) VALUES(?1,?2,'line',1)",
|
||||
)?;
|
||||
for id in start..(start + 1000).min(files) {
|
||||
let extension = if id.is_multiple_of(2) { "txt" } else { "rs" };
|
||||
let filename = format!("record_{id:07}.{extension}");
|
||||
let path = root
|
||||
.join(if id % 3 == 0 {
|
||||
"chosen"
|
||||
} else {
|
||||
"chosen-other"
|
||||
})
|
||||
.join(&filename);
|
||||
let path = path.to_string_lossy();
|
||||
let content = if id < docs {
|
||||
format!("Synthetic document uniquemarker{id:07} cancellation renewal notice category{} seed{SEED}", id % 97)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
insert.execute(params![
|
||||
path,
|
||||
root.to_string_lossy(),
|
||||
filename,
|
||||
content,
|
||||
(id % 8192 + 100) as i64,
|
||||
(1700000000000i64 + id as i64) * 1_000_000,
|
||||
extension
|
||||
])?;
|
||||
if !content.is_empty() {
|
||||
passage.execute(params![content, path])?;
|
||||
}
|
||||
}
|
||||
}
|
||||
transaction.commit()?;
|
||||
}
|
||||
for table in ["search", "names", "content_search", "passage_search"] {
|
||||
// content_search intentionally omits empty files; external-content
|
||||
// comparison would incorrectly require those omitted row IDs.
|
||||
let rank = if table == "content_search" { 0 } else { 1 };
|
||||
connection
|
||||
.execute_batch(&format!(
|
||||
"INSERT INTO {table}({table},rank) VALUES('integrity-check',{rank});"
|
||||
))
|
||||
.with_context(|| format!("FTS integrity check for {table}"))?;
|
||||
}
|
||||
let integrity: String = connection.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
if integrity != "ok" {
|
||||
bail!("SQLite integrity check: {integrity}");
|
||||
}
|
||||
let counts: (usize, usize, usize) = connection.query_row(
|
||||
"SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM files WHERE content!=''),(SELECT count(*) FROM passages)",
|
||||
[], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
||||
)?;
|
||||
if counts != (files, docs, docs) {
|
||||
bail!("seed row counts differ: {counts:?}");
|
||||
}
|
||||
connection.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run(index: &SearchIndex, case: &str, id: usize, root: &Path, debug: bool) -> Result<Value> {
|
||||
let mut options = SearchOptions::default();
|
||||
let query = match case {
|
||||
"exact" => format!(
|
||||
"record_{id:07}.{}",
|
||||
if id.is_multiple_of(2) { "txt" } else { "rs" }
|
||||
),
|
||||
"keyword_unique" => format!("uniquemarker{id:07}"),
|
||||
"common_keyword" => "cancellation".into(),
|
||||
"common_name" => "record_".into(),
|
||||
"strict_filters" => {
|
||||
options.folder = Some(root.join("chosen"));
|
||||
options.extension = Some("rs".into());
|
||||
options.code_only = true;
|
||||
options.min_size = Some(110);
|
||||
options.max_size = Some(119);
|
||||
options.modified_after_unix_ms = Some(1700000000010);
|
||||
options.modified_before_unix_ms = Some(1700000000019);
|
||||
"cancellation".into()
|
||||
}
|
||||
"absent" => "nonexistentzzxyartifact".into(),
|
||||
_ => bail!("unknown case"),
|
||||
};
|
||||
let start = Instant::now();
|
||||
let response = index.search_with_options_measured(&query, LIMIT, &options, debug)?;
|
||||
let wall_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||
if case == "absent" {
|
||||
if !response.results.is_empty() {
|
||||
bail!("absent query returned results");
|
||||
}
|
||||
} else {
|
||||
let first = response
|
||||
.results
|
||||
.first()
|
||||
.with_context(|| format!("positive {case} query returned nothing: {query}"))?;
|
||||
if case == "exact" && (first.filename != query || first.match_kind != "exact_filename") {
|
||||
bail!("exact filename priority failed");
|
||||
}
|
||||
if case == "keyword_unique" && !first.filename.starts_with(&format!("record_{id:07}.")) {
|
||||
bail!("unique keyword identity failed");
|
||||
}
|
||||
if case == "strict_filters"
|
||||
&& (response.results.len() != 1 || first.filename != "record_0000015.rs")
|
||||
{
|
||||
bail!("strict filter expected exactly the seeded eligible record 15");
|
||||
}
|
||||
for result in &response.results {
|
||||
if case == "strict_filters"
|
||||
&& (!Path::new(&result.path).starts_with(root.join("chosen"))
|
||||
|| !result.filename.ends_with(".rs"))
|
||||
{
|
||||
bail!("strict folder/code/extension filter failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug != response.metrics.trace.is_some() {
|
||||
bail!("trace mode mismatch");
|
||||
}
|
||||
Ok(json!({"wall_ms":wall_ms,"metrics":response.metrics}))
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<String> = env::args().skip(1).collect();
|
||||
if args.len() != 4 {
|
||||
bail!("usage: searchi-scale-eval FILES DOCUMENTS SAMPLES REPORT.json");
|
||||
}
|
||||
let files: usize = args[0].parse()?;
|
||||
let docs: usize = args[1].parse()?;
|
||||
let samples: usize = args[2].parse()?;
|
||||
if !(100..=1000000).contains(&files)
|
||||
|| !(20..=100000).contains(&docs)
|
||||
|| docs > files
|
||||
|| !(20..=1000).contains(&samples)
|
||||
{
|
||||
bail!("require 100..1M files, 20..100K docs <= files, 20..1000 samples");
|
||||
}
|
||||
// TempDir owns exactly this directory. No caller directory is recursively removed.
|
||||
let owned = tempfile::Builder::new()
|
||||
.prefix("searchi-scale-")
|
||||
.tempdir()?;
|
||||
let root = owned.path().join("corpus");
|
||||
for folder in ["chosen", "chosen-other"] {
|
||||
fs::create_dir_all(root.join(folder))?;
|
||||
}
|
||||
let root = fs::canonicalize(root)?;
|
||||
let db = owned.path().join("index.sqlite");
|
||||
let seeding = Instant::now();
|
||||
seed(&db, &root, files, docs)?;
|
||||
let seed_ms = seeding.elapsed().as_secs_f64() * 1000.0;
|
||||
let first_open = Instant::now();
|
||||
let mut index = SearchIndex::open(&db)?;
|
||||
let open_ms = first_open.elapsed().as_secs_f64() * 1000.0;
|
||||
let unprepared_first = run(&index, "exact", files / 2, &root, false)?;
|
||||
let prepare = Instant::now();
|
||||
let cached_frequencies = index.prewarm()?;
|
||||
let prepare_ms = prepare.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut cases = Vec::new();
|
||||
for case in [
|
||||
"exact",
|
||||
"keyword_unique",
|
||||
"common_keyword",
|
||||
"common_name",
|
||||
"strict_filters",
|
||||
"absent",
|
||||
] {
|
||||
let first = run(&index, case, docs / 2 + 1, &root, false)?;
|
||||
let mut basic = Vec::with_capacity(samples);
|
||||
let mut debug = Vec::with_capacity(samples);
|
||||
for sample in 0..samples {
|
||||
// Paired AB/BA to avoid systematically favoring one trace mode.
|
||||
let id = (sample * 7919 + SEED as usize) % docs;
|
||||
for detailed in if sample.is_multiple_of(2) {
|
||||
[false, true]
|
||||
} else {
|
||||
[true, false]
|
||||
} {
|
||||
let value = run(&index, case, id, &root, detailed)?;
|
||||
if detailed {
|
||||
debug.push(value);
|
||||
} else {
|
||||
basic.push(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
cases.push(json!({"case":case,"first_after_prewarm":first,"basic":basic,"debug":debug}));
|
||||
}
|
||||
drop(index);
|
||||
let restart = Instant::now();
|
||||
let reopened = SearchIndex::open(&db)?;
|
||||
let reopened_open_ms = restart.elapsed().as_secs_f64() * 1000.0;
|
||||
let reopened_first = run(&reopened, "keyword_unique", docs - 1, &root, false)?;
|
||||
drop(reopened);
|
||||
let connection = Connection::open(&db)?;
|
||||
let schema: i64 = connection.query_row("PRAGMA user_version", [], |r| r.get(0))?;
|
||||
let plan: Vec<String> = connection
|
||||
.prepare(
|
||||
"EXPLAIN QUERY PLAN SELECT id FROM files WHERE lower(filename)='record_0000001.rs'",
|
||||
)?
|
||||
.query_map([], |r| r.get(3))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
connection.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")?;
|
||||
drop(connection);
|
||||
let report = json!({"kind":"synthetic_lexical_index_only","seed":SEED,"files":files,"documents":docs,"schema":schema,
|
||||
"os":env::consts::OS,"arch":env::consts::ARCH,"logical_cpus":std::thread::available_parallelism()?.get(),
|
||||
"executable_sha256":hash(&env::current_exe()?)?,"seed_ms":seed_ms,"database_bytes":fs::metadata(&db)?.len(),
|
||||
"database_sha256":hash(&db)?,"first_reader_open_ms":open_ms,"unprepared_first_input":unprepared_first,
|
||||
"prewarm_ms":prepare_ms,"cached_frequencies":cached_frequencies,"reopened_open_ms":reopened_open_ms,
|
||||
"reopened_first_input":reopened_first,"exact_lookup_plan":plan,"cases":cases,
|
||||
"cold_storage_tested":false,"physical_files_created":0,"inference":false});
|
||||
// create_new prevents accidental replacement of previous evidence.
|
||||
let output = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&args[3])?;
|
||||
serde_json::to_writer_pretty(output, &report)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Summarize retained raw samples; nearest-rank p95, no gate claims."""
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
|
||||
|
||||
def percentile(values, probability):
|
||||
if not values or not 0 < probability <= 1:
|
||||
raise ValueError("nonempty samples and probability in (0, 1] required")
|
||||
if any(not math.isfinite(value) or value < 0 for value in values):
|
||||
raise ValueError("samples must be finite nonnegative milliseconds")
|
||||
ordered = sorted(values)
|
||||
return ordered[math.ceil(len(ordered) * probability) - 1]
|
||||
|
||||
|
||||
def summarize(report):
|
||||
if report.get("kind") != "synthetic_lexical_index_only":
|
||||
raise ValueError("unsupported report")
|
||||
rows = []
|
||||
for case in report["cases"]:
|
||||
if len(case["basic"]) != len(case["debug"]) or not case["basic"]:
|
||||
raise ValueError("paired basic/debug samples required")
|
||||
row = {"case": case["case"], "samples_per_mode": len(case["basic"])}
|
||||
for mode in ("basic", "debug"):
|
||||
values = [sample["wall_ms"] for sample in case[mode]]
|
||||
row[mode] = {
|
||||
"p50_ms": percentile(values, 0.5),
|
||||
"p95_ms": percentile(values, 0.95),
|
||||
"maximum_ms": max(values),
|
||||
"engine_p95_ms": percentile(
|
||||
[sample["metrics"]["total_ms"] for sample in case[mode]], 0.95
|
||||
),
|
||||
}
|
||||
# Pair order follows the same deterministic request IDs in both arrays.
|
||||
row["mean_paired_debug_minus_basic_ms"] = sum(
|
||||
debug["wall_ms"] - basic["wall_ms"]
|
||||
for basic, debug in zip(case["basic"], case["debug"])
|
||||
) / len(case["basic"])
|
||||
rows.append(row)
|
||||
return {"kind": report["kind"], "production_gate_pass": None, "cases": rows}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
print(json.dumps(summarize(json.load(source)), indent=2))
|
||||
@@ -0,0 +1,30 @@
|
||||
import unittest
|
||||
from summarize import percentile, summarize
|
||||
|
||||
|
||||
class SummaryTests(unittest.TestCase):
|
||||
def test_nearest_rank_and_unsorted_input(self):
|
||||
self.assertEqual(percentile(list(range(20, 0, -1)), 0.95), 19)
|
||||
self.assertEqual(percentile([3, 1, 2], 0.5), 2)
|
||||
|
||||
def test_reject_invalid_samples(self):
|
||||
for values in ([], [-1], [float("nan")], [float("inf")]):
|
||||
with self.assertRaises(ValueError):
|
||||
percentile(values, 0.95)
|
||||
|
||||
def test_paired_overhead_retains_negative_noise(self):
|
||||
sample = lambda ms: {"wall_ms": ms, "metrics": {"total_ms": ms / 2}}
|
||||
report = {"kind": "synthetic_lexical_index_only", "cases": [{
|
||||
"case": "exact", "basic": [sample(4), sample(6)],
|
||||
"debug": [sample(3), sample(5)],
|
||||
}]}
|
||||
result = summarize(report)
|
||||
self.assertIsNone(result["production_gate_pass"])
|
||||
self.assertEqual(result["cases"][0]["mean_paired_debug_minus_basic_ms"], -1)
|
||||
report["cases"][0]["debug"].pop()
|
||||
with self.assertRaises(ValueError):
|
||||
summarize(report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user