perf: audit EmbeddingGemma 2 and reuse measured query embeddings
Core CI / core (ubuntu-latest) (push) Failing after 3m43s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 5m56s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
No files matched your search
@@ -11,6 +11,8 @@ import {
|
||||
AutomaticIndexGate,
|
||||
emptySearchState,
|
||||
LatestLane,
|
||||
semanticAvailable,
|
||||
SEMANTIC_SETTLE_MS,
|
||||
} from "./search-state";
|
||||
import type { SemanticPhase } from "./search-state";
|
||||
import { fileIcon, icon } from "./icons";
|
||||
@@ -272,12 +274,7 @@ function refreshEmptySearchState() {
|
||||
}
|
||||
|
||||
function canSearchSemantically() {
|
||||
return Boolean(
|
||||
semanticStatus?.ready &&
|
||||
semanticStatus.generation !== null &&
|
||||
!semanticStatus.embedding &&
|
||||
!semanticJobActive,
|
||||
);
|
||||
return semanticAvailable(semanticStatus);
|
||||
}
|
||||
|
||||
function queueSemanticRefresh(delay = 1000) {
|
||||
@@ -954,7 +951,7 @@ function refreshPerformance(request: SearchRequest) {
|
||||
);
|
||||
for (const [lane, delay] of Object.entries(report.lane_schedule_ms))
|
||||
append(
|
||||
`${lane}: scheduled ${delay.toFixed(1)} ms after debounce${lane === "hybrid" ? " (includes intentional 350 ms semantic delay or waiting for readiness)" : ""}`,
|
||||
`${lane}: scheduled ${delay.toFixed(1)} ms after debounce${lane === "hybrid" ? ` (includes intentional ${SEMANTIC_SETTLE_MS} ms semantic delay or waiting for readiness)` : ""}`,
|
||||
);
|
||||
for (const attempt of report.attempts) {
|
||||
append(
|
||||
@@ -1123,7 +1120,7 @@ async function search(inputAt = performance.now()) {
|
||||
hybridTimer = setTimeout(() => {
|
||||
if (isCurrent(request) && canSearchSemantically())
|
||||
submitSearch(request, "hybrid");
|
||||
}, 350);
|
||||
}, SEMANTIC_SETTLE_MS);
|
||||
} catch (error) {
|
||||
showError(error);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,14 @@
|
||||
/** Let rapid typing settle before starting the model lane. */
|
||||
export const SEMANTIC_SETTLE_MS = 120;
|
||||
|
||||
/** A rebuild retains the previous committed generation. Backend readiness is
|
||||
* withdrawn only while a replacement query graph is being prepared. */
|
||||
export function semanticAvailable(
|
||||
status: { ready: boolean; generation: number | null } | null,
|
||||
) {
|
||||
return Boolean(status?.ready && status.generation !== null);
|
||||
}
|
||||
|
||||
/** One active request per lane; intermediate keystrokes collapse to the latest.
|
||||
* Native work already running cannot be cancelled, so revisions guard every
|
||||
* result and retry. Busy responses receive at most one delayed retry. */
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { LatestLane, AutomaticIndexGate, emptySearchState } from '../src/search-state.ts';
|
||||
import { LatestLane, AutomaticIndexGate, emptySearchState, semanticAvailable, SEMANTIC_SETTLE_MS } from '../src/search-state.ts';
|
||||
const settle = () => new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
test('coalesces queued keystrokes and discards stale responses', async () => {
|
||||
@@ -67,3 +67,14 @@ test('no document content and unavailable semantics have actionable distinct gui
|
||||
assert.match(emptySearchState('lexical', 'no-content').detail, /readable documents/);
|
||||
assert.match(emptySearchState('lexical', 'unavailable').detail, /retry the search/);
|
||||
});
|
||||
|
||||
|
||||
test('old committed semantic generation stays available during rebuild, activation readiness gates replacement', () => {
|
||||
assert.equal(semanticAvailable(null), false);
|
||||
assert.equal(semanticAvailable({ready: true, generation: null, embedding: true}), false);
|
||||
assert.equal(semanticAvailable({ready: true, generation: 1, embedding: false}), true);
|
||||
assert.equal(semanticAvailable({ready: true, generation: 1, embedding: true}), true);
|
||||
assert.equal(semanticAvailable({ready: false, generation: 2, embedding: true}), false);
|
||||
assert.equal(semanticAvailable({ready: true, generation: 2, embedding: false}), true);
|
||||
assert.equal(SEMANTIC_SETTLE_MS, 120);
|
||||
});
|
||||
@@ -28,7 +28,6 @@ use tokio_util::codec::{FramedRead, FramedWrite};
|
||||
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
||||
struct SearchInput {
|
||||
/// Filename, content or remembered meaning. At most 4096 UTF-8 bytes.
|
||||
query: String,
|
||||
@@ -71,7 +70,6 @@ struct State {
|
||||
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
||||
struct SourceInput {
|
||||
handle: String,
|
||||
}
|
||||
|
||||
@@ -13,10 +13,14 @@ pub use searchi_core::{SearchMetrics, SearchOptions, SearchResponse, SearchResul
|
||||
pub use searchi_semantic::WorkerInfo;
|
||||
use searchi_semantic::{GemmaEmbedder, TextWindow, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
|
||||
use serde::Serialize;
|
||||
use std::{collections::HashMap, path::Path};
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
path::Path,
|
||||
};
|
||||
use store::{Store, MAX_SCAN_CHUNKS};
|
||||
|
||||
const MAX_BATCH: usize = 8;
|
||||
const QUERY_CACHE_ENTRIES: usize = 32;
|
||||
const BYTE_SLICE: usize = 64 * 1024;
|
||||
const BYTE_OVERLAP: usize = 512;
|
||||
const MAX_PASSAGE_WINDOWS: usize = 2048;
|
||||
@@ -135,6 +139,10 @@ pub struct HybridSearch {
|
||||
embedder: Box<dyn EmbeddingProvider>,
|
||||
store: Store,
|
||||
generation_fingerprint: String,
|
||||
// Real provider outputs only, scoped to this model session. Queries never
|
||||
// persist and results/filters are never cached here.
|
||||
query_cache_fingerprint: String,
|
||||
query_cache: VecDeque<(String, Vec<f32>)>,
|
||||
}
|
||||
|
||||
impl HybridSearch {
|
||||
@@ -237,6 +245,8 @@ impl HybridSearch {
|
||||
_ => expected,
|
||||
};
|
||||
Ok(Self {
|
||||
query_cache_fingerprint: embedder.fingerprint().to_owned(),
|
||||
query_cache: VecDeque::new(),
|
||||
lexical,
|
||||
embedder,
|
||||
store,
|
||||
@@ -386,22 +396,7 @@ impl HybridSearch {
|
||||
trace.timed("hybrid.identifier_priority", parent, |_, _| {
|
||||
promote_identifiers(query.trim(), &mut lexical)
|
||||
});
|
||||
let embedding_span = trace.begin("semantic.query_embedding", parent);
|
||||
let vector = if trace.is_debug() {
|
||||
self.embedder.embed_query_debug(query.trim())
|
||||
} else {
|
||||
self.embedder.embed_query(query.trim())
|
||||
};
|
||||
trace.counter(embedding_span, "failed", u64::from(vector.is_err()));
|
||||
trace.end(
|
||||
embedding_span,
|
||||
if trace.is_debug() {
|
||||
self.embedder.query_diagnostics()
|
||||
} else {
|
||||
Default::default()
|
||||
},
|
||||
);
|
||||
let vector = vector?;
|
||||
let vector = self.query_vector(query, trace, parent)?;
|
||||
let semantic =
|
||||
self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?;
|
||||
let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| {
|
||||
@@ -423,9 +418,78 @@ impl HybridSearch {
|
||||
if query.trim().is_empty() || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let vector = self.embedder.embed_query(query.trim())?;
|
||||
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||
let vector = self.query_vector(query, &mut trace, None)?;
|
||||
self.routed_semantic(query, &vector, limit.min(1000), options)
|
||||
}
|
||||
|
||||
fn query_vector(
|
||||
&mut self,
|
||||
query: &str,
|
||||
trace: &mut TraceCollector,
|
||||
parent: Option<SpanToken>,
|
||||
) -> Result<Vec<f32>> {
|
||||
let query = query.trim();
|
||||
ensure!(
|
||||
!query.is_empty() && query.len() <= 4096,
|
||||
"Query must be nonempty and at most 4096 UTF-8 bytes"
|
||||
);
|
||||
let span = trace.begin("semantic.query_embedding", parent);
|
||||
if self.query_cache_fingerprint != self.embedder.fingerprint() {
|
||||
self.query_cache.clear();
|
||||
self.query_cache_fingerprint = self.embedder.fingerprint().to_owned();
|
||||
}
|
||||
if let Some(position) = self
|
||||
.query_cache
|
||||
.iter()
|
||||
.position(|(cached, _)| cached == query)
|
||||
{
|
||||
let entry = self
|
||||
.query_cache
|
||||
.remove(position)
|
||||
.expect("Located cache entry");
|
||||
let vector = entry.1.clone();
|
||||
self.query_cache.push_back(entry);
|
||||
trace.counter(span, "query_cache_hit", 1);
|
||||
trace.counter(span, "query_cache_miss", 0);
|
||||
trace.counter(span, "failed", 0);
|
||||
// No worker was called: never attach its previous diagnostics.
|
||||
trace.end(span, Default::default());
|
||||
return Ok(vector);
|
||||
}
|
||||
trace.counter(span, "query_cache_hit", 0);
|
||||
trace.counter(span, "query_cache_miss", 1);
|
||||
let vector = (|| {
|
||||
let vector = if trace.is_debug() {
|
||||
self.embedder.embed_query_debug(query)?
|
||||
} else {
|
||||
self.embedder.embed_query(query)?
|
||||
};
|
||||
ensure!(
|
||||
vector.len() == self.embedder.dimensions()
|
||||
&& vector.iter().all(|value| value.is_finite())
|
||||
&& vector.iter().any(|value| *value != 0.0),
|
||||
"Provider returned an invalid query vector"
|
||||
);
|
||||
Ok(vector)
|
||||
})();
|
||||
trace.counter(span, "failed", u64::from(vector.is_err()));
|
||||
trace.end(
|
||||
span,
|
||||
if trace.is_debug() {
|
||||
self.embedder.query_diagnostics()
|
||||
} else {
|
||||
Default::default()
|
||||
},
|
||||
);
|
||||
let vector = vector?;
|
||||
if self.query_cache.len() == QUERY_CACHE_ENTRIES {
|
||||
self.query_cache.pop_front();
|
||||
}
|
||||
self.query_cache
|
||||
.push_back((query.to_owned(), vector.clone()));
|
||||
Ok(vector)
|
||||
}
|
||||
}
|
||||
|
||||
fn bounded_title(title: &str) -> &str {
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
//! Cache-control tests count provider calls; they do not measure model relevance.
|
||||
//! Production still loads Google and root runs separate real CPU benchmarks.
|
||||
use anyhow::Result;
|
||||
use searchi_core::{SearchIndex, SearchOptions};
|
||||
use searchi_semantic::TextWindow;
|
||||
use searchi_service::{EmbeddingProvider, HybridSearch};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc, Mutex,
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Calls {
|
||||
queries: Mutex<Vec<String>>,
|
||||
fail: AtomicBool,
|
||||
invalid: AtomicBool,
|
||||
new_fingerprint: AtomicBool,
|
||||
}
|
||||
struct CountingProvider(Arc<Calls>);
|
||||
impl EmbeddingProvider for CountingProvider {
|
||||
fn fingerprint(&self) -> &str {
|
||||
if self.0.new_fingerprint.load(Ordering::Relaxed) {
|
||||
"cache-control-v2"
|
||||
} else {
|
||||
"cache-control-v1"
|
||||
}
|
||||
}
|
||||
fn dimensions(&self) -> usize {
|
||||
768
|
||||
}
|
||||
fn split_windows(&mut self, _: &str, _: &str) -> Result<Vec<TextWindow>> {
|
||||
anyhow::bail!("Not used by cache-control tests")
|
||||
}
|
||||
fn embed_documents(&mut self, _: &[&str], _: &str) -> Result<Vec<Vec<f32>>> {
|
||||
anyhow::bail!("Not used by cache-control tests")
|
||||
}
|
||||
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
|
||||
self.0.queries.lock().unwrap().push(query.into());
|
||||
if self.0.fail.load(Ordering::Relaxed) {
|
||||
anyhow::bail!("Injected provider failure");
|
||||
}
|
||||
let mut vector = vec![0.0; 768];
|
||||
if !self.0.invalid.load(Ordering::Relaxed) {
|
||||
vector[0] = 1.0;
|
||||
}
|
||||
Ok(vector)
|
||||
}
|
||||
fn query_diagnostics(&self) -> BTreeMap<String, u64> {
|
||||
BTreeMap::from([("worker_inference_us".into(), 123)])
|
||||
}
|
||||
}
|
||||
fn fixture() -> Result<(tempfile::TempDir, HybridSearch, Arc<Calls>)> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let database = directory.path().join("index.db");
|
||||
let calls = Arc::new(Calls::default());
|
||||
let service = HybridSearch::with_provider(database, Box::new(CountingProvider(calls.clone())))?;
|
||||
Ok((directory, service, calls))
|
||||
}
|
||||
fn count(calls: &Calls) -> usize {
|
||||
calls.queries.lock().unwrap().len()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_trimmed_queries_reuse_vectors_but_case_and_model_fingerprint_do_not() -> Result<()> {
|
||||
let (_directory, mut service, calls) = fixture()?;
|
||||
let options = SearchOptions::default();
|
||||
service.search(" supplier ", 5, &options)?;
|
||||
service.search("supplier", 5, &options)?;
|
||||
assert_eq!(*calls.queries.lock().unwrap(), vec!["supplier"]);
|
||||
service.search("Supplier", 5, &options)?;
|
||||
assert_eq!(count(&calls), 2);
|
||||
calls.new_fingerprint.store(true, Ordering::Relaxed);
|
||||
service.search("supplier", 5, &options)?;
|
||||
assert_eq!(count(&calls), 3);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filters_and_changed_index_are_recomputed_even_when_vector_is_reused() -> Result<()> {
|
||||
let (directory, mut service, calls) = fixture()?;
|
||||
std::fs::write(directory.path().join("first.txt"), "supplier notice")?;
|
||||
std::fs::write(directory.path().join("second.md"), "supplier notice")?;
|
||||
let mut index = SearchIndex::open(directory.path().join("index.db"))?;
|
||||
index.index_folder(directory.path())?;
|
||||
let txt = SearchOptions {
|
||||
extension: Some("txt".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let md = SearchOptions {
|
||||
extension: Some("md".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
service.search("supplier", 5, &txt)?[0].filename,
|
||||
"first.txt"
|
||||
);
|
||||
assert_eq!(service.search("supplier", 5, &md)?[0].filename, "second.md");
|
||||
assert_eq!(count(&calls), 1);
|
||||
std::fs::remove_file(directory.path().join("first.txt"))?;
|
||||
index.index_folder(directory.path())?;
|
||||
assert!(service.search("supplier", 5, &txt)?.is_empty());
|
||||
assert_eq!(count(&calls), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_hit_trace_never_replays_prior_worker_diagnostics() -> Result<()> {
|
||||
let (_directory, mut service, _calls) = fixture()?;
|
||||
let miss = service.search_measured("supplier", 5, &SearchOptions::default(), true)?;
|
||||
let hit = service.search_measured("supplier", 5, &SearchOptions::default(), true)?;
|
||||
let miss = miss
|
||||
.metrics
|
||||
.trace
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|span| span.name == "semantic.query_embedding")
|
||||
.unwrap();
|
||||
let hit = hit
|
||||
.metrics
|
||||
.trace
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|span| span.name == "semantic.query_embedding")
|
||||
.unwrap();
|
||||
assert_eq!(miss.counters["query_cache_miss"], 1);
|
||||
assert_eq!(miss.counters["worker_inference_us"], 123);
|
||||
assert_eq!(hit.counters["query_cache_hit"], 1);
|
||||
assert_eq!(hit.counters["query_cache_miss"], 0);
|
||||
assert!(!hit.counters.contains_key("worker_inference_us"));
|
||||
assert_eq!(hit.counters["failed"], 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failures_and_invalid_vectors_are_not_cached() -> Result<()> {
|
||||
let (_directory, mut service, calls) = fixture()?;
|
||||
assert!(service
|
||||
.search_semantic(&"界".repeat(2000), 5, &SearchOptions::default())
|
||||
.is_err());
|
||||
assert_eq!(
|
||||
count(&calls),
|
||||
0,
|
||||
"Oversized query must not reach the provider or cache"
|
||||
);
|
||||
calls.fail.store(true, Ordering::Relaxed);
|
||||
for _ in 0..2 {
|
||||
assert!(service
|
||||
.search("supplier", 5, &SearchOptions::default())
|
||||
.is_err());
|
||||
}
|
||||
calls.fail.store(false, Ordering::Relaxed);
|
||||
calls.invalid.store(true, Ordering::Relaxed);
|
||||
for _ in 0..2 {
|
||||
assert!(service
|
||||
.search("supplier", 5, &SearchOptions::default())
|
||||
.is_err());
|
||||
}
|
||||
calls.invalid.store(false, Ordering::Relaxed);
|
||||
service.search("supplier", 5, &SearchOptions::default())?;
|
||||
service.search("supplier", 5, &SearchOptions::default())?;
|
||||
assert_eq!(count(&calls), 5);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_evicts_least_recently_used_query_at_32_entries() -> Result<()> {
|
||||
let (_directory, mut service, calls) = fixture()?;
|
||||
for number in 0..32 {
|
||||
service.search(&format!("query{number}"), 5, &SearchOptions::default())?;
|
||||
}
|
||||
service.search("query0", 5, &SearchOptions::default())?;
|
||||
service.search("query32", 5, &SearchOptions::default())?;
|
||||
service.search("query0", 5, &SearchOptions::default())?;
|
||||
assert_eq!(count(&calls), 33);
|
||||
service.search("query1", 5, &SearchOptions::default())?;
|
||||
assert_eq!(count(&calls), 34);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warmup_bypasses_cache_and_never_seeds_an_unseen_measured_query() -> Result<()> {
|
||||
let (_directory, mut service, calls) = fixture()?;
|
||||
service.warmup()?;
|
||||
assert_eq!(count(&calls), 1);
|
||||
let first = service.search_measured(
|
||||
"warm up local file search",
|
||||
5,
|
||||
&SearchOptions::default(),
|
||||
true,
|
||||
)?;
|
||||
assert_eq!(count(&calls), 2);
|
||||
let span = first
|
||||
.metrics
|
||||
.trace
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|span| span.name == "semantic.query_embedding")
|
||||
.unwrap();
|
||||
assert_eq!(span.counters["query_cache_miss"], 1);
|
||||
service.warmup()?;
|
||||
assert_eq!(count(&calls), 3);
|
||||
service.search("warm up local file search", 5, &SearchOptions::default())?;
|
||||
assert_eq!(count(&calls), 3);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
# EmbeddingGemma 2: efficiency and performance audit
|
||||
|
||||
Research and Windows CPU measurements, 2026-10-12. **The required Google model
|
||||
is integrated correctly, but the surrounding pipeline still has substantial
|
||||
efficiency and media-quality work.** This audit keeps fresh inference, cached
|
||||
query vectors, indexing throughput and user-visible delays separate.
|
||||
|
||||
## What is correct
|
||||
|
||||
The pinned model is `google/embeddinggemma-2`, with Q8_0 backbone/projector
|
||||
artifacts verified by size and SHA-256. It is not the older text-only
|
||||
EmbeddingGemma or cloud Gemini Embedding 2. Our runtime uses its dedicated
|
||||
architecture, bidirectional encoding, mean pooling and L2-normalized 768d output.
|
||||
Text-only query workers retain the model; media projector initialization is lazy.
|
||||
No language-model generation or network round trip is involved in search.
|
||||
|
||||
Our search-query and titled-document prefixes match Google's instructions.
|
||||
Overlapping 2,048-token document windows are a deliberate runtime policy below
|
||||
the model's 8,192-token limit; verified byte offsets retain the passage tail.
|
||||
Audio is mono 16 kHz. Native runtime parity was independently checked earlier;
|
||||
that does not establish broad retrieval quality.
|
||||
[Google's model card and guidance](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2),
|
||||
[local adapter](../crates/searchi-semantic/src/gemma.rs),
|
||||
[native worker](../crates/searchi-semantic/native/worker.cpp).
|
||||
|
||||
## Actual CPU measurements
|
||||
|
||||
Host: Windows x86-64 VM, four exposed Ryzen 7 7800X3D logical CPUs and about
|
||||
9.9 GiB RAM. No GPU inference, rendering or media decoding was used. Native
|
||||
worker is a Release build; MCP profile is identified separately below.
|
||||
OS file pages may already be warm. Tests used public/generated fixture content.
|
||||
|
||||
### Fresh model inference
|
||||
|
||||
Eight text queries, three repetitions each, with real Google vectors checked
|
||||
for finite values, 768 dimensions and unit length. Each request still runs the
|
||||
encoder; this table has no query cache.
|
||||
|
||||
| CPU threads | Warm p50 | Warm p95 | First query after worker starts |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| 1 | 74.90 ms | 86.59 ms | 113.29 ms |
|
||||
| 2 | 40.80 ms | 46.66 ms | 64.25 ms |
|
||||
| 4 | 34.16 ms | 49.86 ms | 49.98 ms |
|
||||
|
||||
Includes Python NDJSON encode/decode and worker inference; excludes the Rust
|
||||
adapter, ANN, UI and media encoders. Thread configurations ran sequentially,
|
||||
not randomized. These 24 samples per setting are a diagnostic, not a production
|
||||
p95 guarantee. Two threads had the best observed tail; four improved the median
|
||||
and document throughput but did not improve this query tail.
|
||||
[Raw samples and worker counters](evidence/embeddinggemma2-native-cpu.json).
|
||||
|
||||
Direct worker process startup took 653–816 ms; a separate verified backbone
|
||||
checksum took 170 ms in Python. These are not the whole desktop startup.
|
||||
Actual Release MCP startup through verified assets, shared service and real
|
||||
warmup took **4.88 seconds**. Its first unseen foreground search took **81.95 ms**.
|
||||
This exposes an additional startup gap to profile, without attributing it to an
|
||||
unmeasured stage or weakening artifact verification.
|
||||
[Release shared-service sample](evidence/embeddinggemma2-mcp-release.json).
|
||||
|
||||
### Repeated searches and filter changes
|
||||
|
||||
We added a 32-entry LRU of **actual query vectors**, scoped to the retained
|
||||
provider/fingerprint. Exact trimmed queries reuse their vector. Filters, source
|
||||
freshness, ANN retrieval and ranking run again; complete search results are not
|
||||
cached. The cache is bounded, in memory only, and never seeded with the warmup
|
||||
query to disguise first use. Debug traces distinguish hits from real inference.
|
||||
|
||||
Same seven-file corpus, native CPU worker, one thread and Rust Debug executable:
|
||||
|
||||
| Case | Before mean | After mean | Samples per run |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| Repeat an already searched query | 71.19 ms | 1.20 ms | 8 |
|
||||
| Change filters for that query | 70.94 ms | 1.07 ms | 4 |
|
||||
|
||||
All **20 compared result sets** retained identical filenames, source locations,
|
||||
snippets, scores and order, including seven different descriptive queries.
|
||||
The uncached first query still invoked Google: 82.99 ms before, 77.17 ms after;
|
||||
that small difference is not an optimization claim. The later Release run
|
||||
measured 0.75 ms mean for repeats and 0.70 ms for filter changes. There is no
|
||||
paired old-Release run, so do not call this a Release before/after speedup.
|
||||
[Before](evidence/embeddinggemma2-mcp-before.json),
|
||||
[after](evidence/embeddinggemma2-mcp-after.json).
|
||||
|
||||
The old CLI binary hash was not captured; its source checkpoint was `9870df7`.
|
||||
The new reports include executable hashes. No other inference benchmark was
|
||||
intentionally run concurrently, but compilation/background VM work can contend.
|
||||
|
||||
### Indexing is a different workload
|
||||
|
||||
Eight short documents in one native IPC request took 3.64 s / 1.87 s / 1.24 s
|
||||
with 1 / 2 / 4 threads. Eight separate requests took 3.84 s / 1.89 s / 1.24 s.
|
||||
The single/batch vector cosine minimum was 0.999999997. **Our eight-document
|
||||
request is serial inference, not genuine multi-sequence batching.** It mainly
|
||||
reduces IPC overhead. These are single comparisons, not throughput distributions.
|
||||
|
||||
Earlier independent media indexing took 668 seconds for 20 assets/118 chunks on
|
||||
one CPU thread. Frame/recording costs and quality cannot be inferred from fast
|
||||
text queries. Precise spoken-topic localization, music confusion and no-answer
|
||||
handling remain failed gates; the earlier experimental media rating is still
|
||||
4/10. This cache change does not improve that score.
|
||||
[Actual media evidence](MEDIA-QUALITY.md).
|
||||
|
||||
## Changes made from this audit
|
||||
|
||||
- Added the bounded real-vector cache, with failure/eviction/fingerprint,
|
||||
filtering/source lifecycle and trace tests.
|
||||
- Reduced the intentional semantic UI delay from 350 to 120 ms; the separate
|
||||
60 ms lexical debounce and bounded request lanes remain. That removes 230 ms
|
||||
of configured waiting, not a measured 230 ms compositor improvement.
|
||||
- Kept an existing ready semantic generation searchable during rebuilds.
|
||||
Initial indexing still needs a generation; activation/warmup still withdraws
|
||||
backend readiness. Foreground/indexer CPU contention is not solved by this.
|
||||
- Updated capture validation to use an unseen query when recording actual
|
||||
worker-inference counters, rather than mislabeling a cache hit.
|
||||
|
||||
## Highest-impact next work
|
||||
|
||||
| Priority | Evidence / proposed work | Required acceptance evidence |
|
||||
| --- | --- | --- |
|
||||
| P0 | Bound the whole IPC exchange: synchronous writes currently precede the response timeout. | Nonreading-helper regression; large request deadline and recovery. |
|
||||
| P0 | Decide the x86 CPU baseline: current static build enables AVX2 without runtime CPU variants. | Declared minimum ISA or tested baseline/AVX2 dispatch; older-CPU launch. |
|
||||
| P1 | Reuse unchanged document/media vectors; replacement builds currently re-encode all sources. | One-file change embeds only affected chunks; atomic activation and stale/deletion tests. |
|
||||
| P1 | Pack independent short sequences by token budget; the native worker currently encodes serially. | Vector isolation/parity, length-bucketed CPU/GPU throughput and memory comparisons. |
|
||||
| P1 | Give interactive queries a shared resource budget/priority during indexing. | First-unseen and warm p95 with real background document/media jobs. |
|
||||
| P1 | Profile startup stages and compare a smaller query-only context. | Verification/load/graph-warmup/RSS measured separately; no fake first-query cache. |
|
||||
| P2 | Buffer bounded IPC reads, remove unnecessary stream tying, and make window offset work linear. | Protocol bounds, UTF-8 offsets/tails and real media/payload timing parity. |
|
||||
| P2 | Consider modality-selective projector loading; current lazy combined projector loads both encoders. | Real text/image/audio RSS and load measurements on supported backends. |
|
||||
|
||||
[DigUp's pinned runtime](https://github.com/ARahim3/DigUp/blob/6faa29e4c315a3a591de1b41bd78751bdf39588d/Sources/LlamaRuntime/LlamaEmbedder.swift)
|
||||
demonstrates independent-sequence packing and a smaller text-query context.
|
||||
Those are useful patterns, not proof of an identical speedup on this VM.
|
||||
|
||||
Google supports re-normalized MRL truncation to 512/256/128d. A 256d experiment
|
||||
could reduce vector storage threefold; it does **not** eliminate the model's
|
||||
forward pass. Keep 768d until independent retrieval tests pass. Higher vision
|
||||
budgets trade cost for detail, so retain content-specific quality experiments.
|
||||
[Google's truncation and modality guidance](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2).
|
||||
|
||||
GPU execution remains untested here. Audit its precision against Google's
|
||||
BF16/FP32 guidance, and move the inference timing boundary past asynchronous
|
||||
embedding synchronization before interpreting GPU stage traces. CPU-only results
|
||||
do not validate accelerator throughput, quality or failure recovery.
|
||||
|
||||
## Reproduce
|
||||
|
||||
The benchmark requires already verified model assets, a compiled CPU worker and
|
||||
an explicitly selected fixture index. It does not download or index user files.
|
||||
Read its scope fields and raw samples before comparing hardware or profiles.
|
||||
|
||||
```powershell
|
||||
python tools/embeddinggemma2/benchmark.py --mode native --models .searchi/models/embeddinggemma2 --worker .searchi/runtime/cpu/Release/searchi-gemma-worker.exe --threads 1 2 4 --repeats 3 --output .searchi/native-performance.json
|
||||
python tools/embeddinggemma2/benchmark.py --mode mcp --models .searchi/models/embeddinggemma2 --worker .searchi/runtime/cpu/Release/searchi-gemma-worker.exe --cli target/release/searchi.exe --db .searchi/desktop-capture/index.db --repeats 8 --rust-profile release --output .searchi/mcp-performance.json
|
||||
```
|
||||
@@ -96,6 +96,16 @@ real Google inference remains required for product semantic search. Platform,
|
||||
large-corpus, concurrent-indexing and GPU performance gates remain independent.
|
||||
## Worker round-trip diagnostics
|
||||
|
||||
The retained hybrid service now keeps a bounded 32-query vector LRU. Debug
|
||||
`semantic.query_embedding` spans report `query_cache_hit` and `query_cache_miss`.
|
||||
A hit reuses an actual model vector and performs no neural inference; it must
|
||||
not carry the previous worker's tokenization/inference counters. Filters, current
|
||||
source eligibility, ANN retrieval and fusion still execute. Warmup performs an
|
||||
actual neural probe and does not seed an unseen foreground-query result.
|
||||
Compare fresh queries, repeats and filter changes separately. The current
|
||||
[EmbeddingGemma 2 audit](EMBEDDINGGEMMA2-AUDIT.md) includes paired real-CPU
|
||||
measurements, startup boundaries and remaining indexing/media costs.
|
||||
|
||||
Measured desktop failures return `{message, metrics}` and retain the trace for
|
||||
the failed attempt. Busy retries are separate attempts. If an unexpected worker
|
||||
task failure loses its collector, elapsed time is retained, the trace is null,
|
||||
|
||||
@@ -80,6 +80,29 @@ Status: implementation started; no production release yet. Windows VM tests use
|
||||
- [ ] Native UI interaction and chosen-provider quality/latency checks before
|
||||
claiming production-ready answers or compatibility with every provider.
|
||||
|
||||
## 7. EmbeddingGemma 2 efficiency
|
||||
|
||||
Audit and measured priorities: [EMBEDDINGGEMMA2-AUDIT.md](EMBEDDINGGEMMA2-AUDIT.md).
|
||||
|
||||
- [x] Audit actual required model, task prefixes, pooling/normalization, native
|
||||
sequence processing and CPU/runtime policies against primary sources.
|
||||
- [x] Record real CPU 1/2/4-thread query/first-use/document measurements and
|
||||
paired repeated-query/filter-change results with unchanged retrieval evidence.
|
||||
- [x] Bounded provider-scoped actual-query vector LRU with accurate cache traces;
|
||||
preserve filtering, source freshness, failures and actual warmup probes.
|
||||
- [x] Reduce semantic settle delay to 120 ms and retain old ready generation
|
||||
during rebuilding; test initial-generation and activation readiness boundaries.
|
||||
- [ ] Bound whole IPC write/read deadlines and recover from a nonreading helper.
|
||||
- [ ] Declare/test x86 CPU minimum or baseline/AVX2 runtime selection.
|
||||
- [ ] Incrementally reuse unchanged document/media vectors in atomic generations.
|
||||
- [ ] Benchmark genuine token-budgeted multi-sequence batching and isolation.
|
||||
- [ ] Implement and measure shared foreground/indexer resource budgets/priority.
|
||||
- [ ] Attribute startup cost; measure smaller query context and selective encoders.
|
||||
- [ ] Buffered bounded media IPC and linear UTF-8 source-window offset processing.
|
||||
- [ ] Evaluate 256d MRL and content-specific vision budgets on independent data.
|
||||
- [ ] Validate GPU precision, synchronization-aware timings, parity and fallback
|
||||
on actual accelerator hardware; CPU VM checks do not satisfy this gate.
|
||||
|
||||
Completion requires recorded evidence; targets are not measured results.
|
||||
|
||||
## Completed foundation details
|
||||
|
||||
@@ -10,7 +10,7 @@ and `SEARCHI_FORCE_CPU=1`.
|
||||
- Native desktop: **13/13 tests passed**, including structured trace/basic responses
|
||||
and four optional-answer source/consent/revision tests.
|
||||
Post-generation cached query warmup is implemented but its interactive first-query gate remains open.
|
||||
Frontend: **35/35 tests passed**, TypeScript/Vite production build and
|
||||
Frontend: **36/36 tests passed**, TypeScript/Vite production build and
|
||||
formatting passed. Preview policy tests cover fractional timestamp seeking,
|
||||
image labels, one active/latest pending request, stale replies and Escape.
|
||||
- Semantic adapter: **8 default tests passed**; the explicit 48 MP resource test
|
||||
@@ -43,8 +43,8 @@ and `SEARCHI_FORCE_CPU=1`.
|
||||
that run: [recorded benchmark](../crates/searchi-core/benchmarks/results/windows-cpu-full-candidate-first-final.json).
|
||||
These measurements do not establish every query or concurrent-indexing goal.
|
||||
|
||||
The integrated CPU workspace checkpoint passed **72 default tests** (7 optional AI,
|
||||
8 assets, 8 CLI/MCP, 24 core, 4 media, 8 semantic, 13 service), with **10 explicit
|
||||
The integrated CPU workspace checkpoint passed **78 default tests** (7 optional AI,
|
||||
8 assets, 8 CLI/MCP, 24 core, 4 media, 8 semantic, 19 service), with **10 explicit
|
||||
opt-in tests ignored**. Workspace and native all-target Clippy passed with
|
||||
warnings denied. The later semantic CPU-restart counter change passed workspace
|
||||
Clippy and the actual CPU query diagnostics test. Earlier aggregate counts are historical.
|
||||
@@ -128,6 +128,18 @@ agent execution and release packaging remain open gates.
|
||||
|
||||
## Historical evidence
|
||||
|
||||
The [EmbeddingGemma 2 audit](EMBEDDINGGEMMA2-AUDIT.md) records new CPU diagnostics:
|
||||
24 uncached native queries per 1/2/4-thread setting, paired real MCP repeat/filter
|
||||
measurements, 20 unchanged result-set comparisons, actual Release startup/first
|
||||
query, and document single-versus-serial-batch parity. Six cache-control tests
|
||||
verify eviction, fingerprint/trim behavior, failures, unchanged filtering/source
|
||||
lifecycle and accurate hit traces. The frontend readiness regression retains an
|
||||
old committed generation during rebuilds and blocks initial/activation states.
|
||||
The four real-engine browser captures were regenerated and visually inspected;
|
||||
the performance capture deliberately uses an unseen query and proves actual
|
||||
worker inference rather than treating a cached vector as a new neural pass.
|
||||
Native visual/paint and production media relevance gates remain open.
|
||||
|
||||
Initial engine/CLI checks covered retrieval, persistence, filesystem changes,
|
||||
exclusions/filters, migration, concurrent readers, PDF pages, DOCX paragraphs,
|
||||
corrupt replacement, encrypted PDF errors and bounded compressed input.
|
||||
|
||||
@@ -0,0 +1,730 @@
|
||||
{
|
||||
"captured_at_utc": "2026-10-12T18:41:00Z",
|
||||
"os": "Windows-11-10.0.26300-SP0",
|
||||
"machine": "AMD64",
|
||||
"logical_cpus": 4,
|
||||
"cpu_only": true,
|
||||
"rust_profile": "debug",
|
||||
"label": "bounded-query-vector-cache",
|
||||
"worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3",
|
||||
"cli_sha256": "4a4c69c34f48162e2033fbfbc7b3500aa7496f6c1d9a03be6d611cf84bafe6e0",
|
||||
"results": {
|
||||
"startup_to_initialized_ms": 11040.071900002658,
|
||||
"stats": {
|
||||
"files": {
|
||||
"content_files": 5,
|
||||
"files": 7
|
||||
},
|
||||
"mode": "hybrid",
|
||||
"semantic_available": true,
|
||||
"worker": {
|
||||
"device": "cpu",
|
||||
"dimensions": 768,
|
||||
"media_supported": true,
|
||||
"model": "google/embeddinggemma-2",
|
||||
"ready": true,
|
||||
"runtime": "b11461-no-logits-v1",
|
||||
"threads": 1
|
||||
}
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"case": "first_unseen",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 79.04949999647215,
|
||||
"engine_ms": 77.1732,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.6412000006530434,
|
||||
"engine_ms": 1.4406,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.3130000045057386,
|
||||
"engine_ms": 1.1769,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.4286000116262585,
|
||||
"engine_ms": 1.2572,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.217100001871586,
|
||||
"engine_ms": 1.0070000000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.3689999943599105,
|
||||
"engine_ms": 1.0303,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.5655000063125044,
|
||||
"engine_ms": 1.1261,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.9205000028014183,
|
||||
"engine_ms": 1.395,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 2.2301000135485083,
|
||||
"engine_ms": 1.1747999999999998,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "pdf",
|
||||
"request_ms": 1.8029999919235706,
|
||||
"engine_ms": 0.975,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "md",
|
||||
"request_ms": 2.3480000090785325,
|
||||
"engine_ms": 1.0859,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "pdf",
|
||||
"request_ms": 2.0995999802835286,
|
||||
"engine_ms": 1.0337,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "md",
|
||||
"request_ms": 2.343600004678592,
|
||||
"engine_ms": 1.1913,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "mountains reflected in calm water",
|
||||
"extension": null,
|
||||
"request_ms": 75.07760002044961,
|
||||
"engine_ms": 73.8136,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Mountain Lake.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "die Kündigungsfrist im Lieferantenvertrag",
|
||||
"extension": null,
|
||||
"request_ms": 89.54819999053143,
|
||||
"engine_ms": 86.77550000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "source code that checks indexed file permissions",
|
||||
"extension": null,
|
||||
"request_ms": 77.97480002045631,
|
||||
"engine_ms": 76.4551,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "music with piano and a slow melody",
|
||||
"extension": null,
|
||||
"request_ms": 78.43809999758378,
|
||||
"engine_ms": 76.8372,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "a boat crossing a lake at sunset",
|
||||
"extension": null,
|
||||
"request_ms": 77.43000000482425,
|
||||
"engine_ms": 75.7792,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Mountain Lake.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "notes about returning unused materials",
|
||||
"extension": null,
|
||||
"request_ms": 70.03259999328293,
|
||||
"engine_ms": 68.49170000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "find the recording discussing orange soil",
|
||||
"extension": null,
|
||||
"request_ms": 78.86860001599416,
|
||||
"engine_ms": 77.1636,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"same_query": {
|
||||
"samples": 8,
|
||||
"mean_ms": 1.2009875,
|
||||
"p50_ms": 1.17585,
|
||||
"p95_ms": 1.4406
|
||||
},
|
||||
"filter_change": {
|
||||
"samples": 4,
|
||||
"mean_ms": 1.071475,
|
||||
"p50_ms": 1.0598,
|
||||
"p95_ms": 1.1913
|
||||
},
|
||||
"scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
{
|
||||
"captured_at_utc": "2026-10-12T18:35:22Z",
|
||||
"os": "Windows-11-10.0.26300-SP0",
|
||||
"machine": "AMD64",
|
||||
"logical_cpus": 4,
|
||||
"cpu_only": true,
|
||||
"rust_profile": "debug",
|
||||
"results": {
|
||||
"startup_to_initialized_ms": 7216.115900024306,
|
||||
"stats": {
|
||||
"files": {
|
||||
"content_files": 5,
|
||||
"files": 7
|
||||
},
|
||||
"mode": "hybrid",
|
||||
"semantic_available": true,
|
||||
"worker": {
|
||||
"device": "cpu",
|
||||
"dimensions": 768,
|
||||
"media_supported": true,
|
||||
"model": "google/embeddinggemma-2",
|
||||
"ready": true,
|
||||
"runtime": "b11461-no-logits-v1",
|
||||
"threads": 1
|
||||
}
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"case": "first_unseen",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 85.7454999932088,
|
||||
"engine_ms": 82.9858,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 84.06170000671409,
|
||||
"engine_ms": 82.0945,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 69.04040000517853,
|
||||
"engine_ms": 67.36319999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 66.24039998860098,
|
||||
"engine_ms": 64.4344,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 67.91299997712485,
|
||||
"engine_ms": 66.0718,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 70.40650001727045,
|
||||
"engine_ms": 68.77619999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 79.71600000746548,
|
||||
"engine_ms": 77.87230000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 69.87529998878017,
|
||||
"engine_ms": 68.0868,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 77.02430000063032,
|
||||
"engine_ms": 74.836,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "pdf",
|
||||
"request_ms": 84.02040001237765,
|
||||
"engine_ms": 79.5709,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "md",
|
||||
"request_ms": 72.45289999991655,
|
||||
"engine_ms": 70.70519999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "pdf",
|
||||
"request_ms": 68.47949998336844,
|
||||
"engine_ms": 66.9529,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "md",
|
||||
"request_ms": 68.12429998535663,
|
||||
"engine_ms": 66.5463,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "mountains reflected in calm water",
|
||||
"extension": null,
|
||||
"request_ms": 74.91510000545532,
|
||||
"engine_ms": 73.53999999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Mountain Lake.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "die Kündigungsfrist im Lieferantenvertrag",
|
||||
"extension": null,
|
||||
"request_ms": 84.24939998076297,
|
||||
"engine_ms": 82.6408,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "source code that checks indexed file permissions",
|
||||
"extension": null,
|
||||
"request_ms": 79.35359998373315,
|
||||
"engine_ms": 77.66290000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "music with piano and a slow melody",
|
||||
"extension": null,
|
||||
"request_ms": 80.26409999001771,
|
||||
"engine_ms": 78.6324,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "a boat crossing a lake at sunset",
|
||||
"extension": null,
|
||||
"request_ms": 79.87940000020899,
|
||||
"engine_ms": 78.0203,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Mountain Lake.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "notes about returning unused materials",
|
||||
"extension": null,
|
||||
"request_ms": 73.2068000070285,
|
||||
"engine_ms": 71.48490000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "find the recording discussing orange soil",
|
||||
"extension": null,
|
||||
"request_ms": 79.31890001054853,
|
||||
"engine_ms": 77.6251,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"same_query": {
|
||||
"samples": 8,
|
||||
"mean_ms": 71.1919,
|
||||
"p50_ms": 68.4315,
|
||||
"p95_ms": 82.0945
|
||||
},
|
||||
"filter_change": {
|
||||
"samples": 4,
|
||||
"mean_ms": 70.943825,
|
||||
"p50_ms": 68.82905,
|
||||
"p95_ms": 79.5709
|
||||
},
|
||||
"scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint."
|
||||
},
|
||||
"label": "baseline-no-query-vector-cache",
|
||||
"worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3",
|
||||
"cli_sha256": null,
|
||||
"measurement_base_commit": "9870df7",
|
||||
"metadata_note": "Worker unchanged during these runs; hash recorded afterward. Baseline CLI executable hash was not captured; source checkpoint was 9870df7."
|
||||
}
|
||||
@@ -0,0 +1,730 @@
|
||||
{
|
||||
"captured_at_utc": "2026-10-12T18:47:54Z",
|
||||
"os": "Windows-11-10.0.26300-SP0",
|
||||
"machine": "AMD64",
|
||||
"logical_cpus": 4,
|
||||
"cpu_only": true,
|
||||
"rust_profile": "release",
|
||||
"label": "release-query-vector-cache",
|
||||
"worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3",
|
||||
"cli_sha256": "9e6c087c863b606d02534bff2db1eb397df3a2b8c9aa2f875b2e5ac0a2d04316",
|
||||
"results": {
|
||||
"startup_to_initialized_ms": 4878.2795000006445,
|
||||
"stats": {
|
||||
"files": {
|
||||
"content_files": 5,
|
||||
"files": 7
|
||||
},
|
||||
"mode": "hybrid",
|
||||
"semantic_available": true,
|
||||
"worker": {
|
||||
"device": "cpu",
|
||||
"dimensions": 768,
|
||||
"media_supported": true,
|
||||
"model": "google/embeddinggemma-2",
|
||||
"ready": true,
|
||||
"runtime": "b11461-no-logits-v1",
|
||||
"threads": 1
|
||||
}
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"case": "first_unseen",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 82.9809999850113,
|
||||
"engine_ms": 81.94659999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.6709000046830624,
|
||||
"engine_ms": 0.777,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.6380999877583236,
|
||||
"engine_ms": 0.9057,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.4798000047449023,
|
||||
"engine_ms": 0.6547000000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.6269000188913196,
|
||||
"engine_ms": 0.8484999999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.3796999992337078,
|
||||
"engine_ms": 0.6398,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.5811999910511076,
|
||||
"engine_ms": 0.8515,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.2971000105608255,
|
||||
"engine_ms": 0.6551,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "same_query",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": null,
|
||||
"request_ms": 1.2206999817863107,
|
||||
"engine_ms": 0.7019,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "pdf",
|
||||
"request_ms": 1.7168000049423426,
|
||||
"engine_ms": 0.6491,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "md",
|
||||
"request_ms": 1.3719999988097697,
|
||||
"engine_ms": 0.6258,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "pdf",
|
||||
"request_ms": 1.505599997472018,
|
||||
"engine_ms": 0.6131,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "filter_change",
|
||||
"query": "cancel the supplier partnership",
|
||||
"extension": "md",
|
||||
"request_ms": 1.6737999976612628,
|
||||
"engine_ms": 0.9121,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "mountains reflected in calm water",
|
||||
"extension": null,
|
||||
"request_ms": 77.39190000575036,
|
||||
"engine_ms": 75.18950000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Mountain Lake.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "die Kündigungsfrist im Lieferantenvertrag",
|
||||
"extension": null,
|
||||
"request_ms": 98.24019999359734,
|
||||
"engine_ms": 97.0811,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "source code that checks indexed file permissions",
|
||||
"extension": null,
|
||||
"request_ms": 91.89109998987988,
|
||||
"engine_ms": 90.8722,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "music with piano and a slow melody",
|
||||
"extension": null,
|
||||
"request_ms": 85.49989998573437,
|
||||
"engine_ms": 84.59949999999999,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "a boat crossing a lake at sunset",
|
||||
"extension": null,
|
||||
"request_ms": 86.64829999906942,
|
||||
"engine_ms": 85.5861,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Night Pier.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Mountain Lake.jpg",
|
||||
"source": {
|
||||
"kind": "image",
|
||||
"number": 0
|
||||
},
|
||||
"snippet": "Image content",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Travel Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "notes about returning unused materials",
|
||||
"extension": null,
|
||||
"request_ms": 69.34839999303222,
|
||||
"engine_ms": 68.37740000000001,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Supplier Agreement.pdf",
|
||||
"source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "distinct_query",
|
||||
"query": "find the recording discussing orange soil",
|
||||
"extension": null,
|
||||
"request_ms": 79.9731999868527,
|
||||
"engine_ms": 78.8852,
|
||||
"matches": [
|
||||
{
|
||||
"filename": "Garden Notes.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
|
||||
"score": 0.01639344262295082
|
||||
},
|
||||
{
|
||||
"filename": "Q3 Procurement.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
|
||||
"score": 0.016129032258064516
|
||||
},
|
||||
{
|
||||
"filename": "Vendor Onboarding.md",
|
||||
"source": {
|
||||
"kind": "line",
|
||||
"number": 1
|
||||
},
|
||||
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
|
||||
"score": 0.015873015873015872
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"same_query": {
|
||||
"samples": 8,
|
||||
"mean_ms": 0.754275,
|
||||
"p50_ms": 0.7394499999999999,
|
||||
"p95_ms": 0.9057
|
||||
},
|
||||
"filter_change": {
|
||||
"samples": 4,
|
||||
"mean_ms": 0.700025,
|
||||
"p50_ms": 0.6374500000000001,
|
||||
"p95_ms": 0.9121
|
||||
},
|
||||
"scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint."
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"kind": "browser-rendered-production-ui-real-cpu-engine",
|
||||
"source": {
|
||||
"base_commit": "b8400c4fbf4038820c383ed429207a3b339bf0a5",
|
||||
"base_commit": "9870df7cb5b537ba516997a0d049b996bc30b08d",
|
||||
"working_tree_dirty": true
|
||||
},
|
||||
"viewport": {
|
||||
@@ -37,7 +37,7 @@
|
||||
"metrics": {
|
||||
"dropped_spans": 0,
|
||||
"result_count": 7,
|
||||
"total_ms": 84.7475,
|
||||
"total_ms": 2.0688999999999997,
|
||||
"trace": null
|
||||
},
|
||||
"first_filename": "Supplier Agreement.pdf",
|
||||
@@ -58,7 +58,7 @@
|
||||
"metrics": {
|
||||
"dropped_spans": 0,
|
||||
"result_count": 7,
|
||||
"total_ms": 83.50019999999999,
|
||||
"total_ms": 1.5953,
|
||||
"trace": null
|
||||
},
|
||||
"first_filename": "Mountain Lake.jpg",
|
||||
@@ -71,183 +71,185 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"query": "cancel the supplier partnership",
|
||||
"query": "how can we cancel the supplier partnership",
|
||||
"mode": "debug",
|
||||
"result_count": 7,
|
||||
"metrics": {
|
||||
"dropped_spans": 0,
|
||||
"result_count": 7,
|
||||
"total_ms": 81.5728,
|
||||
"total_ms": 92.8279,
|
||||
"trace": [
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 81.5544,
|
||||
"duration_ms": 92.80539999999999,
|
||||
"name": "hybrid.search",
|
||||
"parent": null,
|
||||
"start_ms": 0.0106
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 1.5203,
|
||||
"duration_ms": 2.6003,
|
||||
"name": "lexical.search",
|
||||
"parent": 0,
|
||||
"start_ms": 0.0281
|
||||
"start_ms": 0.027
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"query_bytes": 31
|
||||
"query_bytes": 42
|
||||
},
|
||||
"duration_ms": 0.0383,
|
||||
"duration_ms": 0.0389,
|
||||
"name": "lexical.normalization",
|
||||
"parent": 1,
|
||||
"start_ms": 0.0386
|
||||
"start_ms": 0.0374
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"frequency_cache_hits": 29,
|
||||
"frequency_cache_misses": 0,
|
||||
"frequency_cache_hits": 30,
|
||||
"frequency_cache_misses": 8,
|
||||
"trigram_index": 1
|
||||
},
|
||||
"duration_ms": 0.1389,
|
||||
"duration_ms": 1.0081,
|
||||
"name": "lexical.candidate_plan",
|
||||
"parent": 1,
|
||||
"start_ms": 0.0871
|
||||
"start_ms": 0.08650000000000001
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.1952,
|
||||
"duration_ms": 0.2286,
|
||||
"name": "lexical.exact_filename",
|
||||
"parent": 1,
|
||||
"start_ms": 0.2358
|
||||
"start_ms": 1.1168
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filters_active": 0,
|
||||
"rows": 0
|
||||
},
|
||||
"duration_ms": 0.16620000000000001,
|
||||
"duration_ms": 0.1837,
|
||||
"name": "lexical.filtered_sql",
|
||||
"parent": 4,
|
||||
"start_ms": 0.254
|
||||
"start_ms": 1.1471
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.183,
|
||||
"duration_ms": 0.2688,
|
||||
"name": "lexical.filename_substring",
|
||||
"parent": 1,
|
||||
"start_ms": 0.44539999999999996
|
||||
"start_ms": 1.3663
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filters_active": 0,
|
||||
"rows": 0
|
||||
},
|
||||
"duration_ms": 0.15589999999999998,
|
||||
"duration_ms": 0.2249,
|
||||
"name": "lexical.filtered_sql",
|
||||
"parent": 6,
|
||||
"start_ms": 0.4609
|
||||
"start_ms": 1.3906999999999998
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.1605,
|
||||
"duration_ms": 0.2146,
|
||||
"name": "lexical.path_substring",
|
||||
"parent": 1,
|
||||
"start_ms": 0.642
|
||||
"start_ms": 1.6556
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filters_active": 0,
|
||||
"rows": 0
|
||||
},
|
||||
"duration_ms": 0.1332,
|
||||
"duration_ms": 0.175,
|
||||
"name": "lexical.filtered_sql",
|
||||
"parent": 8,
|
||||
"start_ms": 0.6575
|
||||
"start_ms": 1.6802
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.1908,
|
||||
"duration_ms": 0.2391,
|
||||
"name": "lexical.content",
|
||||
"parent": 1,
|
||||
"start_ms": 0.8159000000000001
|
||||
"start_ms": 1.8861999999999999
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filters_active": 0,
|
||||
"rows": 0
|
||||
},
|
||||
"duration_ms": 0.1622,
|
||||
"duration_ms": 0.2053,
|
||||
"name": "lexical.filtered_sql",
|
||||
"parent": 10,
|
||||
"start_ms": 0.8313
|
||||
"start_ms": 1.9049
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.21509999999999999,
|
||||
"duration_ms": 0.3054,
|
||||
"name": "lexical.filename_terms",
|
||||
"parent": 1,
|
||||
"start_ms": 1.0205
|
||||
"start_ms": 2.1413
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filters_active": 0,
|
||||
"rows": 0
|
||||
},
|
||||
"duration_ms": 0.1858,
|
||||
"duration_ms": 0.2773,
|
||||
"name": "lexical.filtered_sql",
|
||||
"parent": 12,
|
||||
"start_ms": 1.0362
|
||||
"start_ms": 2.1576999999999997
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.27109999999999995,
|
||||
"duration_ms": 0.146,
|
||||
"name": "lexical.media_metadata",
|
||||
"parent": 1,
|
||||
"start_ms": 1.2537
|
||||
"start_ms": 2.4598
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filters_active": 0,
|
||||
"rows": 0
|
||||
},
|
||||
"duration_ms": 0.2419,
|
||||
"duration_ms": 0.112,
|
||||
"name": "lexical.filtered_sql",
|
||||
"parent": 14,
|
||||
"start_ms": 1.2715999999999998
|
||||
"start_ms": 2.4749
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.0155,
|
||||
"name": "hybrid.identifier_priority",
|
||||
"parent": 0,
|
||||
"start_ms": 1.5598999999999998
|
||||
"start_ms": 2.6382
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"failed": 0,
|
||||
"request_encode_us": 20,
|
||||
"request_write_us": 54,
|
||||
"response_validate_us": 94,
|
||||
"worker_batch_prepare_us": 16,
|
||||
"worker_inference_us": 77535,
|
||||
"worker_pool_normalize_us": 18,
|
||||
"worker_token_count": 13,
|
||||
"worker_tokenization_us": 146,
|
||||
"worker_wait_and_response_decode_us": 78781
|
||||
"query_cache_hit": 0,
|
||||
"query_cache_miss": 1,
|
||||
"request_encode_us": 18,
|
||||
"request_write_us": 58,
|
||||
"response_validate_us": 69,
|
||||
"worker_batch_prepare_us": 19,
|
||||
"worker_inference_us": 87311,
|
||||
"worker_pool_normalize_us": 14,
|
||||
"worker_token_count": 16,
|
||||
"worker_tokenization_us": 173,
|
||||
"worker_wait_and_response_decode_us": 88454
|
||||
},
|
||||
"duration_ms": 79.06530000000001,
|
||||
"duration_ms": 89.1821,
|
||||
"name": "semantic.query_embedding",
|
||||
"parent": 0,
|
||||
"start_ms": 1.5843999999999998
|
||||
"start_ms": 2.6649
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"general_lane": 1
|
||||
},
|
||||
"duration_ms": 0.8277,
|
||||
"duration_ms": 0.8757,
|
||||
"name": "semantic.routing",
|
||||
"parent": 0,
|
||||
"start_ms": 80.6811
|
||||
"start_ms": 91.8792
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
@@ -255,99 +257,99 @@
|
||||
"filesystem_checks": 2,
|
||||
"result_files": 7
|
||||
},
|
||||
"duration_ms": 0.7831,
|
||||
"duration_ms": 0.8334,
|
||||
"name": "semantic.retrieval",
|
||||
"parent": 18,
|
||||
"start_ms": 80.71549999999999
|
||||
"start_ms": 91.90950000000001
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.1451,
|
||||
"duration_ms": 0.1384,
|
||||
"name": "semantic.generation_snapshot",
|
||||
"parent": 19,
|
||||
"start_ms": 80.73429999999999
|
||||
"start_ms": 91.9247
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"cache_hit": 1
|
||||
},
|
||||
"duration_ms": 0.017499999999999998,
|
||||
"duration_ms": 0.0115,
|
||||
"name": "semantic.ann_load",
|
||||
"parent": 19,
|
||||
"start_ms": 80.89150000000001
|
||||
"start_ms": 92.07310000000001
|
||||
},
|
||||
{
|
||||
"counters": {},
|
||||
"duration_ms": 0.0108,
|
||||
"duration_ms": 0.011,
|
||||
"name": "semantic.filter_normalization",
|
||||
"parent": 19,
|
||||
"start_ms": 80.9172
|
||||
"start_ms": 92.0933
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"cache_hit": 1,
|
||||
"eligible_chunks": 8
|
||||
},
|
||||
"duration_ms": 0.0245,
|
||||
"duration_ms": 0.0252,
|
||||
"name": "semantic.eligibility",
|
||||
"parent": 19,
|
||||
"start_ms": 80.93599999999999
|
||||
"start_ms": 92.1129
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"requested_chunks": 8
|
||||
},
|
||||
"duration_ms": 0.49450000000000005,
|
||||
"duration_ms": 0.5494,
|
||||
"name": "semantic.ann_pass",
|
||||
"parent": 19,
|
||||
"start_ms": 80.9745
|
||||
"start_ms": 92.1485
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"returned_chunks": 8
|
||||
},
|
||||
"duration_ms": 0.036000000000000004,
|
||||
"duration_ms": 0.0347,
|
||||
"name": "semantic.ann_search",
|
||||
"parent": 24,
|
||||
"start_ms": 80.9829
|
||||
"start_ms": 92.161
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"filesystem_checks": 2,
|
||||
"unique_parents": 7
|
||||
},
|
||||
"duration_ms": 0.4331,
|
||||
"duration_ms": 0.4802,
|
||||
"name": "semantic.parent_materialization",
|
||||
"parent": 24,
|
||||
"start_ms": 81.0271
|
||||
"start_ms": 92.2066
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"valid": 1
|
||||
},
|
||||
"duration_ms": 0.1684,
|
||||
"duration_ms": 0.20550000000000002,
|
||||
"name": "semantic.parent_filesystem_validation",
|
||||
"parent": 26,
|
||||
"start_ms": 81.1383
|
||||
"start_ms": 92.30539999999999
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"valid": 1
|
||||
},
|
||||
"duration_ms": 0.0964,
|
||||
"duration_ms": 0.1146,
|
||||
"name": "semantic.parent_filesystem_validation",
|
||||
"parent": 26,
|
||||
"start_ms": 81.333
|
||||
"start_ms": 92.5374
|
||||
},
|
||||
{
|
||||
"counters": {
|
||||
"lexical_files": 0,
|
||||
"semantic_files": 7
|
||||
},
|
||||
"duration_ms": 0.0385,
|
||||
"duration_ms": 0.0376,
|
||||
"name": "hybrid.rank_fusion",
|
||||
"parent": 0,
|
||||
"start_ms": 81.5188
|
||||
"start_ms": 92.7663
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -355,18 +357,13 @@
|
||||
"first_source": {
|
||||
"kind": "page",
|
||||
"number": 2
|
||||
},
|
||||
"ui_checks": {
|
||||
"literal_query_marks": true,
|
||||
"actual_page_explanation": true,
|
||||
"why_disclosure_collapsed_by_default": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"screenshots": [
|
||||
{
|
||||
"file": "search-light.png",
|
||||
"sha256": "a8347729d7995d0354fd45c1d0762d3fe33adbf0dfcf9ac3387073e522710b15",
|
||||
"sha256": "bd45041de13093f7a7e26285618f4c1afc5b081fb41e51ff06e1313216af3562",
|
||||
"query": "cancel the supplier partnership",
|
||||
"mode": "basic",
|
||||
"result_count": 7,
|
||||
@@ -374,7 +371,7 @@
|
||||
"first_source_kind": "page",
|
||||
"first_source_number": 2,
|
||||
"theme": "light",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 85 MS · SHOWN 543 MS",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 2.1 MS · SHOWN 233 MS",
|
||||
"state_assertions": {
|
||||
"semantic_ready": true,
|
||||
"hybrid_final": true,
|
||||
@@ -387,7 +384,7 @@
|
||||
},
|
||||
{
|
||||
"file": "search-dark.png",
|
||||
"sha256": "b8cf9505ada4afb439edd469d58868fff92dd99cf9b62b6d4a5127307c1e36a7",
|
||||
"sha256": "cd7e2aa3f7027cd9b40965171a232f6ee901e1e2b971f04628b4cfcc3b440263",
|
||||
"query": "cancel the supplier partnership",
|
||||
"mode": "basic",
|
||||
"result_count": 7,
|
||||
@@ -395,7 +392,7 @@
|
||||
"first_source_kind": "page",
|
||||
"first_source_number": 2,
|
||||
"theme": "dark",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 85 MS · SHOWN 543 MS",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 2.1 MS · SHOWN 233 MS",
|
||||
"state_assertions": {
|
||||
"semantic_ready": true,
|
||||
"hybrid_final": true,
|
||||
@@ -408,7 +405,7 @@
|
||||
},
|
||||
{
|
||||
"file": "media-search.png",
|
||||
"sha256": "db832104e3b2fc59bd07a4ea393d38d363c00385023e5f9e469726e076e9b9fe",
|
||||
"sha256": "4c278446963cd5dbad073bea93a3ca33fb9c10cb43a9544258d5fb6039014d81",
|
||||
"query": "mountains reflected in calm water",
|
||||
"mode": "basic",
|
||||
"result_count": 7,
|
||||
@@ -416,7 +413,7 @@
|
||||
"first_source_kind": "image",
|
||||
"first_source_number": 0,
|
||||
"theme": "light",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 84 MS · SHOWN 519 MS",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 1.6 MS · SHOWN 217 MS",
|
||||
"state_assertions": {
|
||||
"semantic_ready": true,
|
||||
"hybrid_final": true,
|
||||
@@ -427,23 +424,20 @@
|
||||
},
|
||||
{
|
||||
"file": "performance-details.png",
|
||||
"sha256": "149753ca90cda5d81254d43e15118b61325a61d7f0fb366b783d4e2dfac5d96d",
|
||||
"query": "cancel the supplier partnership",
|
||||
"sha256": "4f336d4c42d0c4a50ce33bca27595f02da7d896c8734d447350d50dab5c339df",
|
||||
"query": "how can we cancel the supplier partnership",
|
||||
"mode": "debug",
|
||||
"result_count": 7,
|
||||
"selected_filename": "Supplier Agreement.pdf",
|
||||
"first_source_kind": "page",
|
||||
"first_source_number": 2,
|
||||
"theme": "light",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 82 MS · SHOWN 514 MS",
|
||||
"summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 93 MS · SHOWN 311 MS",
|
||||
"state_assertions": {
|
||||
"semantic_ready": true,
|
||||
"hybrid_final": true,
|
||||
"image_visible": false,
|
||||
"actual_worker_inference": true,
|
||||
"literal_query_marks": true,
|
||||
"actual_page_explanation": true,
|
||||
"why_disclosure_collapsed_by_default": true
|
||||
"actual_worker_inference": true
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
|
Before Width: | Height: | Size: 439 KiB After Width: | Height: | Size: 439 KiB |
|
Before Width: | Height: | Size: 185 KiB After Width: | Height: | Size: 186 KiB |
|
Before Width: | Height: | Size: 168 KiB After Width: | Height: | Size: 168 KiB |
|
Before Width: | Height: | Size: 165 KiB After Width: | Height: | Size: 165 KiB |
@@ -306,10 +306,13 @@ try {
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await screenshot("media-search", imageQuery);
|
||||
await page.locator("#settings summary").click();
|
||||
await page.locator("#settings > summary").click();
|
||||
await page.locator("#performance-toggle").check();
|
||||
await page.locator("#settings summary").click();
|
||||
await query(documentQuery, true);
|
||||
await page.locator("#settings > summary").click();
|
||||
// Use a genuinely unseen query so this screenshot records real inference,
|
||||
// rather than attributing a bounded query-vector cache hit to the worker.
|
||||
const performanceQuery = "how can we cancel the supplier partnership";
|
||||
await query(performanceQuery, true);
|
||||
await page.locator("#performance-panel").waitFor({ state: "visible" });
|
||||
await page.locator("#performance-panel summary").click();
|
||||
await page.waitForFunction(
|
||||
@@ -332,7 +335,7 @@ try {
|
||||
.filter({ hasText: "worker inference" })
|
||||
.first()
|
||||
.scrollIntoViewIfNeeded();
|
||||
await screenshot("performance-details", documentQuery);
|
||||
await screenshot("performance-details", performanceQuery);
|
||||
const semanticResponse = await context.request.post(
|
||||
new URL("/capture-api/semantic_status", target).href,
|
||||
{
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Explicit CPU-only measurements of the real verified Searchi Google runtime.
|
||||
|
||||
Native mode isolates worker IPC/inference. MCP mode measures the actual shared
|
||||
service over stdio against an explicitly supplied, already indexed test corpus.
|
||||
No models are downloaded, files indexed, or production defaults modified.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import queue
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
QUERIES = [
|
||||
'cancel the supplier partnership',
|
||||
'mountains reflected in calm water',
|
||||
'die Kündigungsfrist im Lieferantenvertrag',
|
||||
'source code that checks indexed file permissions',
|
||||
'music with piano and a slow melody',
|
||||
'a boat crossing a lake at sunset',
|
||||
'notes about returning unused materials',
|
||||
'find the recording discussing orange soil',
|
||||
]
|
||||
|
||||
def elapsed(start):
|
||||
return (time.perf_counter() - start) * 1000
|
||||
|
||||
def distribution(values):
|
||||
ordered = sorted(values)
|
||||
return {'samples': len(values), 'mean_ms': statistics.mean(values),
|
||||
'p50_ms': statistics.median(values),
|
||||
'p95_ms': ordered[max(0, math.ceil(len(values) * .95) - 1)]}
|
||||
|
||||
class Session:
|
||||
def __init__(self, command):
|
||||
environment = os.environ.copy()
|
||||
environment['SEARCHI_FORCE_CPU'] = '1'
|
||||
flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0
|
||||
self.started = time.perf_counter()
|
||||
self.process = subprocess.Popen(command, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
encoding='utf-8', env=environment, creationflags=flags)
|
||||
self.lines = queue.Queue()
|
||||
self.errors = []
|
||||
threading.Thread(target=self.read, daemon=True).start()
|
||||
threading.Thread(target=self.read_errors, daemon=True).start()
|
||||
|
||||
def read(self):
|
||||
try:
|
||||
for line in self.process.stdout:
|
||||
self.lines.put(json.loads(line))
|
||||
self.lines.put(RuntimeError('Runtime stdout closed'))
|
||||
except Exception as error:
|
||||
self.lines.put(error)
|
||||
|
||||
def read_errors(self):
|
||||
for line in self.process.stderr:
|
||||
if len(self.errors) < 20:
|
||||
self.errors.append(line.rstrip())
|
||||
|
||||
def next(self):
|
||||
result = self.lines.get(timeout=120)
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
|
||||
def send(self, value):
|
||||
self.process.stdin.write(json.dumps(value, ensure_ascii=False) + '\n')
|
||||
self.process.stdin.flush()
|
||||
|
||||
def worker(self, value):
|
||||
start = time.perf_counter()
|
||||
self.send(value)
|
||||
response = self.next()
|
||||
if not response.get('ok'):
|
||||
raise RuntimeError(response.get('error', 'Worker request failed'))
|
||||
assert response['device'] == 'cpu'
|
||||
return response, elapsed(start)
|
||||
|
||||
def mcp(self, identifier, method, params):
|
||||
start = time.perf_counter()
|
||||
self.send({'jsonrpc': '2.0', 'id': identifier, 'method': method, 'params': params})
|
||||
while True:
|
||||
response = self.next()
|
||||
if response.get('id') == identifier:
|
||||
assert 'error' not in response, response
|
||||
return response['result'], elapsed(start)
|
||||
|
||||
def close(self):
|
||||
self.process.stdin.close()
|
||||
try:
|
||||
self.process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.kill()
|
||||
self.process.wait(timeout=10)
|
||||
|
||||
def vectors(response, count):
|
||||
result = response['vectors']
|
||||
assert len(result) == count
|
||||
for vector in result:
|
||||
assert len(vector) == 768 and all(math.isfinite(n) for n in vector)
|
||||
assert abs(sum(n*n for n in vector) - 1) < .0001
|
||||
return result
|
||||
|
||||
def native(args):
|
||||
manifest = json.loads((ROOT / 'tools/embeddinggemma2/model-manifest.json').read_text())
|
||||
model = args.models / manifest['files'][0]['path']
|
||||
start = time.perf_counter()
|
||||
assert model.stat().st_size == manifest['files'][0]['bytes']
|
||||
with model.open('rb') as stream:
|
||||
assert hashlib.file_digest(stream, 'sha256').hexdigest() == manifest['files'][0]['sha256']
|
||||
verification_ms = elapsed(start)
|
||||
reports = []
|
||||
for threads in args.threads:
|
||||
session = Session([str(args.worker.resolve()), '--model', str(model.resolve()),
|
||||
'--threads', str(threads), '--device', 'cpu'])
|
||||
try:
|
||||
hello = session.next()
|
||||
load_ms = elapsed(session.started)
|
||||
assert hello['device'] == 'cpu' and hello['model'] == manifest['model_id']
|
||||
first, first_ms = session.worker({'op':'query', 'text':'first unseen foreground query',
|
||||
'debug_performance':True})
|
||||
vectors(first, 1)
|
||||
samples = []
|
||||
for repeat in range(args.repeats):
|
||||
for query in QUERIES:
|
||||
response, duration = session.worker({'op':'query','text':query,
|
||||
'debug_performance':True})
|
||||
vectors(response, 1)
|
||||
samples.append({'query':query,'repeat':repeat,'round_trip_ms':duration,
|
||||
'worker':response['performance']})
|
||||
window_samples = []
|
||||
passage = ('Supplier cancellation requires sixty days written notice. '
|
||||
'Kündigung und Rückgabe der Materialien. ') * 500
|
||||
for _ in range(3):
|
||||
response, duration = session.worker({'op':'windows','text':passage,'title':'Agreement'})
|
||||
encoded = passage.encode()
|
||||
for window in response['windows']:
|
||||
assert encoded[window['start_byte']:window['end_byte']].decode() == window['text']
|
||||
assert response['windows'][-1]['end_byte'] == len(encoded)
|
||||
window_samples.append({'bytes':len(encoded),'windows':len(response['windows']),
|
||||
'round_trip_ms':duration})
|
||||
documents = [(f'Document {n}. Supplier cancellation requires sixty days written notice. '
|
||||
'Unused materials must be returned. ') * 5 for n in range(8)]
|
||||
singles = []
|
||||
single_times = []
|
||||
for document in documents:
|
||||
response, duration = session.worker({'op':'documents','texts':[document],'title':'Agreement'})
|
||||
singles.extend(vectors(response, 1))
|
||||
single_times.append(duration)
|
||||
response, batch_ms = session.worker({'op':'documents','texts':documents,'title':'Agreement'})
|
||||
batched = vectors(response, 8)
|
||||
cosines = [sum(a*b for a,b in zip(left,right)) for left,right in zip(singles,batched)]
|
||||
assert min(cosines) > .99999
|
||||
reports.append({'threads':threads,'worker_info':hello,'process_load_ms':load_ms,
|
||||
'first_query_ms':first_ms,'first_query_worker':first['performance'],
|
||||
'warm_queries':distribution([sample['round_trip_ms'] for sample in samples]),
|
||||
'query_samples':samples,'window_samples':window_samples,
|
||||
'documents':{'single_request_ms':single_times,'eight_single_total_ms':sum(single_times),
|
||||
'eight_document_request_ms':batch_ms,'batch_cosine_min':min(cosines)}})
|
||||
print(f'CPU {threads} threads: warm p95 {reports[-1]["warm_queries"]["p95_ms"]:.2f} ms', flush=True)
|
||||
finally:
|
||||
session.close()
|
||||
return {'text_model_verification_ms':verification_ms,'thread_reports':reports,
|
||||
'scope':'Native actual Google Q8 worker; verified model; cached OS pages possible. '
|
||||
'Includes NDJSON encode/decode; excludes Rust adapter, ANN and UI. '
|
||||
'First query precedes warm samples. Documents measure IPC batch, not assumed parallelism.'}
|
||||
|
||||
def mcp(args):
|
||||
assert args.db and args.cli
|
||||
assert args.db.is_file()
|
||||
session = Session([str(args.cli.resolve()),'mcp','--db',str(args.db.resolve()),
|
||||
'--model-dir',str(args.models.resolve()),'--embedding-worker',str(args.worker.resolve()),
|
||||
'--cpu-threads','1'])
|
||||
try:
|
||||
session.mcp(1,'initialize',{'protocolVersion':'2025-11-25','capabilities':{},
|
||||
'clientInfo':{'name':'searchi-cpu-performance','version':'1'}})
|
||||
startup_ms = elapsed(session.started)
|
||||
session.send({'jsonrpc':'2.0','method':'notifications/initialized'})
|
||||
stats,_ = session.mcp(2,'tools/call',{'name':'stats','arguments':{}})
|
||||
assert stats['structuredContent']['mode'] == 'hybrid'
|
||||
assert stats['structuredContent']['worker']['device'] == 'cpu'
|
||||
samples=[]
|
||||
identifier=3
|
||||
cases = [('first_unseen',QUERIES[0],None)]
|
||||
cases += [('same_query',QUERIES[0],None)]*args.repeats
|
||||
cases += [('filter_change',QUERIES[0],ext) for ext in ['pdf','md','pdf','md']]
|
||||
cases += [('distinct_query',query,None) for query in QUERIES[1:]]
|
||||
for case,query,extension in cases:
|
||||
arguments={'query':query,'limit':3}
|
||||
if extension:
|
||||
arguments['extension']=extension
|
||||
result,duration=session.mcp(identifier,'tools/call',{'name':'search','arguments':arguments})
|
||||
identifier+=1
|
||||
assert not result.get('isError'), result
|
||||
response=result['structuredContent']
|
||||
assert response['mode']=='hybrid'
|
||||
if extension:
|
||||
assert all(item['filename'].lower().endswith('.'+extension) for item in response['results'])
|
||||
samples.append({'case':case,'query':query,'extension':extension,'request_ms':duration,
|
||||
'engine_ms':response['metrics']['total_ms'],
|
||||
'matches':[{'filename':item['filename'],'source':item['source'],
|
||||
'snippet':item['snippet'],'score':item['score']}
|
||||
for item in response['results']]})
|
||||
return {'startup_to_initialized_ms':startup_ms,'stats':stats['structuredContent'],
|
||||
'samples':samples,'same_query':distribution([s['engine_ms'] for s in samples if s['case']=='same_query']),
|
||||
'filter_change':distribution([s['engine_ms'] for s in samples if s['case']=='filter_change']),
|
||||
'scope':'Actual shared service through real MCP. Startup includes verified models and background warmup. '
|
||||
'Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint.'}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--models',type=Path,required=True)
|
||||
parser.add_argument('--worker',type=Path,required=True)
|
||||
parser.add_argument('--output',type=Path,required=True)
|
||||
parser.add_argument('--mode',choices=['native','mcp'],default='native')
|
||||
parser.add_argument('--threads',type=int,nargs='+',default=[1,2,4])
|
||||
parser.add_argument('--repeats',type=int,default=3)
|
||||
parser.add_argument('--db',type=Path)
|
||||
parser.add_argument('--cli',type=Path)
|
||||
parser.add_argument('--rust-profile',choices=['debug','release','not-applicable'],default='not-applicable')
|
||||
parser.add_argument('--label',default='')
|
||||
args=parser.parse_args()
|
||||
assert 1<=args.repeats<=20 and all(1<=n<=8 for n in args.threads)
|
||||
def digest(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream,'sha256').hexdigest()
|
||||
report={'captured_at_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),
|
||||
'os':platform.platform(),'machine':platform.machine(),'logical_cpus':os.cpu_count(),
|
||||
'cpu_only':True,'rust_profile':args.rust_profile,'label':args.label,
|
||||
'worker_sha256':digest(args.worker),
|
||||
'cli_sha256':digest(args.cli) if args.cli else None,
|
||||
'results':native(args) if args.mode=='native' else mcp(args)}
|
||||
args.output.parent.mkdir(parents=True,exist_ok=True)
|
||||
args.output.write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
|
||||
print(f'Saved {args.output}',flush=True)
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||