diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index f6538ee..66504da 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -11,6 +11,8 @@ import { AutomaticIndexGate, emptySearchState, LatestLane, + semanticAvailable, + SEMANTIC_SETTLE_MS, } from "./search-state"; import type { SemanticPhase } from "./search-state"; import { fileIcon, icon } from "./icons"; @@ -272,12 +274,7 @@ function refreshEmptySearchState() { } function canSearchSemantically() { - return Boolean( - semanticStatus?.ready && - semanticStatus.generation !== null && - !semanticStatus.embedding && - !semanticJobActive, - ); + return semanticAvailable(semanticStatus); } function queueSemanticRefresh(delay = 1000) { @@ -954,7 +951,7 @@ function refreshPerformance(request: SearchRequest) { ); for (const [lane, delay] of Object.entries(report.lane_schedule_ms)) append( - `${lane}: scheduled ${delay.toFixed(1)} ms after debounce${lane === "hybrid" ? " (includes intentional 350 ms semantic delay or waiting for readiness)" : ""}`, + `${lane}: scheduled ${delay.toFixed(1)} ms after debounce${lane === "hybrid" ? ` (includes intentional ${SEMANTIC_SETTLE_MS} ms semantic delay or waiting for readiness)` : ""}`, ); for (const attempt of report.attempts) { append( @@ -1123,7 +1120,7 @@ async function search(inputAt = performance.now()) { hybridTimer = setTimeout(() => { if (isCurrent(request) && canSearchSemantically()) submitSearch(request, "hybrid"); - }, 350); + }, SEMANTIC_SETTLE_MS); } catch (error) { showError(error); } diff --git a/apps/desktop/src/search-state.ts b/apps/desktop/src/search-state.ts index e638b8b..e1da385 100644 --- a/apps/desktop/src/search-state.ts +++ b/apps/desktop/src/search-state.ts @@ -1,3 +1,14 @@ +/** Let rapid typing settle before starting the model lane. */ +export const SEMANTIC_SETTLE_MS = 120; + +/** A rebuild retains the previous committed generation. Backend readiness is + * withdrawn only while a replacement query graph is being prepared. */ +export function semanticAvailable( + status: { ready: boolean; generation: number | null } | null, +) { + return Boolean(status?.ready && status.generation !== null); +} + /** One active request per lane; intermediate keystrokes collapse to the latest. * Native work already running cannot be cancelled, so revisions guard every * result and retry. Busy responses receive at most one delayed retry. */ diff --git a/apps/desktop/tests/search-state.test.mjs b/apps/desktop/tests/search-state.test.mjs index 998dab0..19f3e4a 100644 --- a/apps/desktop/tests/search-state.test.mjs +++ b/apps/desktop/tests/search-state.test.mjs @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { LatestLane, AutomaticIndexGate, emptySearchState } from '../src/search-state.ts'; +import { LatestLane, AutomaticIndexGate, emptySearchState, semanticAvailable, SEMANTIC_SETTLE_MS } from '../src/search-state.ts'; const settle = () => new Promise(resolve => setImmediate(resolve)); test('coalesces queued keystrokes and discards stale responses', async () => { @@ -67,3 +67,14 @@ test('no document content and unavailable semantics have actionable distinct gui assert.match(emptySearchState('lexical', 'no-content').detail, /readable documents/); assert.match(emptySearchState('lexical', 'unavailable').detail, /retry the search/); }); + + +test('old committed semantic generation stays available during rebuild, activation readiness gates replacement', () => { + assert.equal(semanticAvailable(null), false); + assert.equal(semanticAvailable({ready: true, generation: null, embedding: true}), false); + assert.equal(semanticAvailable({ready: true, generation: 1, embedding: false}), true); + assert.equal(semanticAvailable({ready: true, generation: 1, embedding: true}), true); + assert.equal(semanticAvailable({ready: false, generation: 2, embedding: true}), false); + assert.equal(semanticAvailable({ready: true, generation: 2, embedding: false}), true); + assert.equal(SEMANTIC_SETTLE_MS, 120); +}); diff --git a/crates/searchi-cli/src/mcp.rs b/crates/searchi-cli/src/mcp.rs index 91035a2..b7a8dfd 100644 --- a/crates/searchi-cli/src/mcp.rs +++ b/crates/searchi-cli/src/mcp.rs @@ -28,7 +28,6 @@ use tokio_util::codec::{FramedRead, FramedWrite}; #[derive(Debug, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] - struct SearchInput { /// Filename, content or remembered meaning. At most 4096 UTF-8 bytes. query: String, @@ -71,7 +70,6 @@ struct State { #[derive(Debug, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] - struct SourceInput { handle: String, } diff --git a/crates/searchi-service/src/lib.rs b/crates/searchi-service/src/lib.rs index 40ee60e..2e37f9f 100644 --- a/crates/searchi-service/src/lib.rs +++ b/crates/searchi-service/src/lib.rs @@ -13,10 +13,14 @@ pub use searchi_core::{SearchMetrics, SearchOptions, SearchResponse, SearchResul pub use searchi_semantic::WorkerInfo; use searchi_semantic::{GemmaEmbedder, TextWindow, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT}; use serde::Serialize; -use std::{collections::HashMap, path::Path}; +use std::{ + collections::{HashMap, VecDeque}, + path::Path, +}; use store::{Store, MAX_SCAN_CHUNKS}; const MAX_BATCH: usize = 8; +const QUERY_CACHE_ENTRIES: usize = 32; const BYTE_SLICE: usize = 64 * 1024; const BYTE_OVERLAP: usize = 512; const MAX_PASSAGE_WINDOWS: usize = 2048; @@ -135,6 +139,10 @@ pub struct HybridSearch { embedder: Box, store: Store, generation_fingerprint: String, + // Real provider outputs only, scoped to this model session. Queries never + // persist and results/filters are never cached here. + query_cache_fingerprint: String, + query_cache: VecDeque<(String, Vec)>, } impl HybridSearch { @@ -237,6 +245,8 @@ impl HybridSearch { _ => expected, }; Ok(Self { + query_cache_fingerprint: embedder.fingerprint().to_owned(), + query_cache: VecDeque::new(), lexical, embedder, store, @@ -386,22 +396,7 @@ impl HybridSearch { trace.timed("hybrid.identifier_priority", parent, |_, _| { promote_identifiers(query.trim(), &mut lexical) }); - let embedding_span = trace.begin("semantic.query_embedding", parent); - let vector = if trace.is_debug() { - self.embedder.embed_query_debug(query.trim()) - } else { - self.embedder.embed_query(query.trim()) - }; - trace.counter(embedding_span, "failed", u64::from(vector.is_err())); - trace.end( - embedding_span, - if trace.is_debug() { - self.embedder.query_diagnostics() - } else { - Default::default() - }, - ); - let vector = vector?; + let vector = self.query_vector(query, trace, parent)?; let semantic = self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?; let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| { @@ -423,9 +418,78 @@ impl HybridSearch { if query.trim().is_empty() || limit == 0 { return Ok(Vec::new()); } - let vector = self.embedder.embed_query(query.trim())?; + let mut trace = TraceCollector::new(MetricsMode::Basic); + let vector = self.query_vector(query, &mut trace, None)?; self.routed_semantic(query, &vector, limit.min(1000), options) } + + fn query_vector( + &mut self, + query: &str, + trace: &mut TraceCollector, + parent: Option, + ) -> Result> { + let query = query.trim(); + ensure!( + !query.is_empty() && query.len() <= 4096, + "Query must be nonempty and at most 4096 UTF-8 bytes" + ); + let span = trace.begin("semantic.query_embedding", parent); + if self.query_cache_fingerprint != self.embedder.fingerprint() { + self.query_cache.clear(); + self.query_cache_fingerprint = self.embedder.fingerprint().to_owned(); + } + if let Some(position) = self + .query_cache + .iter() + .position(|(cached, _)| cached == query) + { + let entry = self + .query_cache + .remove(position) + .expect("Located cache entry"); + let vector = entry.1.clone(); + self.query_cache.push_back(entry); + trace.counter(span, "query_cache_hit", 1); + trace.counter(span, "query_cache_miss", 0); + trace.counter(span, "failed", 0); + // No worker was called: never attach its previous diagnostics. + trace.end(span, Default::default()); + return Ok(vector); + } + trace.counter(span, "query_cache_hit", 0); + trace.counter(span, "query_cache_miss", 1); + let vector = (|| { + let vector = if trace.is_debug() { + self.embedder.embed_query_debug(query)? + } else { + self.embedder.embed_query(query)? + }; + ensure!( + vector.len() == self.embedder.dimensions() + && vector.iter().all(|value| value.is_finite()) + && vector.iter().any(|value| *value != 0.0), + "Provider returned an invalid query vector" + ); + Ok(vector) + })(); + trace.counter(span, "failed", u64::from(vector.is_err())); + trace.end( + span, + if trace.is_debug() { + self.embedder.query_diagnostics() + } else { + Default::default() + }, + ); + let vector = vector?; + if self.query_cache.len() == QUERY_CACHE_ENTRIES { + self.query_cache.pop_front(); + } + self.query_cache + .push_back((query.to_owned(), vector.clone())); + Ok(vector) + } } fn bounded_title(title: &str) -> &str { diff --git a/crates/searchi-service/tests/query_cache.rs b/crates/searchi-service/tests/query_cache.rs new file mode 100644 index 0000000..e2cb166 --- /dev/null +++ b/crates/searchi-service/tests/query_cache.rs @@ -0,0 +1,208 @@ +//! Cache-control tests count provider calls; they do not measure model relevance. +//! Production still loads Google and root runs separate real CPU benchmarks. +use anyhow::Result; +use searchi_core::{SearchIndex, SearchOptions}; +use searchi_semantic::TextWindow; +use searchi_service::{EmbeddingProvider, HybridSearch}; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +#[derive(Default)] +struct Calls { + queries: Mutex>, + fail: AtomicBool, + invalid: AtomicBool, + new_fingerprint: AtomicBool, +} +struct CountingProvider(Arc); +impl EmbeddingProvider for CountingProvider { + fn fingerprint(&self) -> &str { + if self.0.new_fingerprint.load(Ordering::Relaxed) { + "cache-control-v2" + } else { + "cache-control-v1" + } + } + fn dimensions(&self) -> usize { + 768 + } + fn split_windows(&mut self, _: &str, _: &str) -> Result> { + anyhow::bail!("Not used by cache-control tests") + } + fn embed_documents(&mut self, _: &[&str], _: &str) -> Result>> { + anyhow::bail!("Not used by cache-control tests") + } + fn embed_query(&mut self, query: &str) -> Result> { + self.0.queries.lock().unwrap().push(query.into()); + if self.0.fail.load(Ordering::Relaxed) { + anyhow::bail!("Injected provider failure"); + } + let mut vector = vec![0.0; 768]; + if !self.0.invalid.load(Ordering::Relaxed) { + vector[0] = 1.0; + } + Ok(vector) + } + fn query_diagnostics(&self) -> BTreeMap { + BTreeMap::from([("worker_inference_us".into(), 123)]) + } +} +fn fixture() -> Result<(tempfile::TempDir, HybridSearch, Arc)> { + let directory = tempfile::tempdir()?; + let database = directory.path().join("index.db"); + let calls = Arc::new(Calls::default()); + let service = HybridSearch::with_provider(database, Box::new(CountingProvider(calls.clone())))?; + Ok((directory, service, calls)) +} +fn count(calls: &Calls) -> usize { + calls.queries.lock().unwrap().len() +} + +#[test] +fn exact_trimmed_queries_reuse_vectors_but_case_and_model_fingerprint_do_not() -> Result<()> { + let (_directory, mut service, calls) = fixture()?; + let options = SearchOptions::default(); + service.search(" supplier ", 5, &options)?; + service.search("supplier", 5, &options)?; + assert_eq!(*calls.queries.lock().unwrap(), vec!["supplier"]); + service.search("Supplier", 5, &options)?; + assert_eq!(count(&calls), 2); + calls.new_fingerprint.store(true, Ordering::Relaxed); + service.search("supplier", 5, &options)?; + assert_eq!(count(&calls), 3); + Ok(()) +} + +#[test] +fn filters_and_changed_index_are_recomputed_even_when_vector_is_reused() -> Result<()> { + let (directory, mut service, calls) = fixture()?; + std::fs::write(directory.path().join("first.txt"), "supplier notice")?; + std::fs::write(directory.path().join("second.md"), "supplier notice")?; + let mut index = SearchIndex::open(directory.path().join("index.db"))?; + index.index_folder(directory.path())?; + let txt = SearchOptions { + extension: Some("txt".into()), + ..Default::default() + }; + let md = SearchOptions { + extension: Some("md".into()), + ..Default::default() + }; + assert_eq!( + service.search("supplier", 5, &txt)?[0].filename, + "first.txt" + ); + assert_eq!(service.search("supplier", 5, &md)?[0].filename, "second.md"); + assert_eq!(count(&calls), 1); + std::fs::remove_file(directory.path().join("first.txt"))?; + index.index_folder(directory.path())?; + assert!(service.search("supplier", 5, &txt)?.is_empty()); + assert_eq!(count(&calls), 1); + Ok(()) +} + +#[test] +fn cache_hit_trace_never_replays_prior_worker_diagnostics() -> Result<()> { + let (_directory, mut service, _calls) = fixture()?; + let miss = service.search_measured("supplier", 5, &SearchOptions::default(), true)?; + let hit = service.search_measured("supplier", 5, &SearchOptions::default(), true)?; + let miss = miss + .metrics + .trace + .unwrap() + .into_iter() + .find(|span| span.name == "semantic.query_embedding") + .unwrap(); + let hit = hit + .metrics + .trace + .unwrap() + .into_iter() + .find(|span| span.name == "semantic.query_embedding") + .unwrap(); + assert_eq!(miss.counters["query_cache_miss"], 1); + assert_eq!(miss.counters["worker_inference_us"], 123); + assert_eq!(hit.counters["query_cache_hit"], 1); + assert_eq!(hit.counters["query_cache_miss"], 0); + assert!(!hit.counters.contains_key("worker_inference_us")); + assert_eq!(hit.counters["failed"], 0); + Ok(()) +} + +#[test] +fn failures_and_invalid_vectors_are_not_cached() -> Result<()> { + let (_directory, mut service, calls) = fixture()?; + assert!(service + .search_semantic(&"界".repeat(2000), 5, &SearchOptions::default()) + .is_err()); + assert_eq!( + count(&calls), + 0, + "Oversized query must not reach the provider or cache" + ); + calls.fail.store(true, Ordering::Relaxed); + for _ in 0..2 { + assert!(service + .search("supplier", 5, &SearchOptions::default()) + .is_err()); + } + calls.fail.store(false, Ordering::Relaxed); + calls.invalid.store(true, Ordering::Relaxed); + for _ in 0..2 { + assert!(service + .search("supplier", 5, &SearchOptions::default()) + .is_err()); + } + calls.invalid.store(false, Ordering::Relaxed); + service.search("supplier", 5, &SearchOptions::default())?; + service.search("supplier", 5, &SearchOptions::default())?; + assert_eq!(count(&calls), 5); + Ok(()) +} + +#[test] +fn cache_evicts_least_recently_used_query_at_32_entries() -> Result<()> { + let (_directory, mut service, calls) = fixture()?; + for number in 0..32 { + service.search(&format!("query{number}"), 5, &SearchOptions::default())?; + } + service.search("query0", 5, &SearchOptions::default())?; + service.search("query32", 5, &SearchOptions::default())?; + service.search("query0", 5, &SearchOptions::default())?; + assert_eq!(count(&calls), 33); + service.search("query1", 5, &SearchOptions::default())?; + assert_eq!(count(&calls), 34); + Ok(()) +} + +#[test] +fn warmup_bypasses_cache_and_never_seeds_an_unseen_measured_query() -> Result<()> { + let (_directory, mut service, calls) = fixture()?; + service.warmup()?; + assert_eq!(count(&calls), 1); + let first = service.search_measured( + "warm up local file search", + 5, + &SearchOptions::default(), + true, + )?; + assert_eq!(count(&calls), 2); + let span = first + .metrics + .trace + .unwrap() + .into_iter() + .find(|span| span.name == "semantic.query_embedding") + .unwrap(); + assert_eq!(span.counters["query_cache_miss"], 1); + service.warmup()?; + assert_eq!(count(&calls), 3); + service.search("warm up local file search", 5, &SearchOptions::default())?; + assert_eq!(count(&calls), 3); + Ok(()) +} diff --git a/docs/EMBEDDINGGEMMA2-AUDIT.md b/docs/EMBEDDINGGEMMA2-AUDIT.md new file mode 100644 index 0000000..2f4b90b --- /dev/null +++ b/docs/EMBEDDINGGEMMA2-AUDIT.md @@ -0,0 +1,153 @@ +# EmbeddingGemma 2: efficiency and performance audit + +Research and Windows CPU measurements, 2026-10-12. **The required Google model +is integrated correctly, but the surrounding pipeline still has substantial +efficiency and media-quality work.** This audit keeps fresh inference, cached +query vectors, indexing throughput and user-visible delays separate. + +## What is correct + +The pinned model is `google/embeddinggemma-2`, with Q8_0 backbone/projector +artifacts verified by size and SHA-256. It is not the older text-only +EmbeddingGemma or cloud Gemini Embedding 2. Our runtime uses its dedicated +architecture, bidirectional encoding, mean pooling and L2-normalized 768d output. +Text-only query workers retain the model; media projector initialization is lazy. +No language-model generation or network round trip is involved in search. + +Our search-query and titled-document prefixes match Google's instructions. +Overlapping 2,048-token document windows are a deliberate runtime policy below +the model's 8,192-token limit; verified byte offsets retain the passage tail. +Audio is mono 16 kHz. Native runtime parity was independently checked earlier; +that does not establish broad retrieval quality. +[Google's model card and guidance](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2), +[local adapter](../crates/searchi-semantic/src/gemma.rs), +[native worker](../crates/searchi-semantic/native/worker.cpp). + +## Actual CPU measurements + +Host: Windows x86-64 VM, four exposed Ryzen 7 7800X3D logical CPUs and about +9.9 GiB RAM. No GPU inference, rendering or media decoding was used. Native +worker is a Release build; MCP profile is identified separately below. +OS file pages may already be warm. Tests used public/generated fixture content. + +### Fresh model inference + +Eight text queries, three repetitions each, with real Google vectors checked +for finite values, 768 dimensions and unit length. Each request still runs the +encoder; this table has no query cache. + +| CPU threads | Warm p50 | Warm p95 | First query after worker starts | +| --- | ---: | ---: | ---: | +| 1 | 74.90 ms | 86.59 ms | 113.29 ms | +| 2 | 40.80 ms | 46.66 ms | 64.25 ms | +| 4 | 34.16 ms | 49.86 ms | 49.98 ms | + +Includes Python NDJSON encode/decode and worker inference; excludes the Rust +adapter, ANN, UI and media encoders. Thread configurations ran sequentially, +not randomized. These 24 samples per setting are a diagnostic, not a production +p95 guarantee. Two threads had the best observed tail; four improved the median +and document throughput but did not improve this query tail. +[Raw samples and worker counters](evidence/embeddinggemma2-native-cpu.json). + +Direct worker process startup took 653–816 ms; a separate verified backbone +checksum took 170 ms in Python. These are not the whole desktop startup. +Actual Release MCP startup through verified assets, shared service and real +warmup took **4.88 seconds**. Its first unseen foreground search took **81.95 ms**. +This exposes an additional startup gap to profile, without attributing it to an +unmeasured stage or weakening artifact verification. +[Release shared-service sample](evidence/embeddinggemma2-mcp-release.json). + +### Repeated searches and filter changes + +We added a 32-entry LRU of **actual query vectors**, scoped to the retained +provider/fingerprint. Exact trimmed queries reuse their vector. Filters, source +freshness, ANN retrieval and ranking run again; complete search results are not +cached. The cache is bounded, in memory only, and never seeded with the warmup +query to disguise first use. Debug traces distinguish hits from real inference. + +Same seven-file corpus, native CPU worker, one thread and Rust Debug executable: + +| Case | Before mean | After mean | Samples per run | +| --- | ---: | ---: | ---: | +| Repeat an already searched query | 71.19 ms | 1.20 ms | 8 | +| Change filters for that query | 70.94 ms | 1.07 ms | 4 | + +All **20 compared result sets** retained identical filenames, source locations, +snippets, scores and order, including seven different descriptive queries. +The uncached first query still invoked Google: 82.99 ms before, 77.17 ms after; +that small difference is not an optimization claim. The later Release run +measured 0.75 ms mean for repeats and 0.70 ms for filter changes. There is no +paired old-Release run, so do not call this a Release before/after speedup. +[Before](evidence/embeddinggemma2-mcp-before.json), +[after](evidence/embeddinggemma2-mcp-after.json). + +The old CLI binary hash was not captured; its source checkpoint was `9870df7`. +The new reports include executable hashes. No other inference benchmark was +intentionally run concurrently, but compilation/background VM work can contend. + +### Indexing is a different workload + +Eight short documents in one native IPC request took 3.64 s / 1.87 s / 1.24 s +with 1 / 2 / 4 threads. Eight separate requests took 3.84 s / 1.89 s / 1.24 s. +The single/batch vector cosine minimum was 0.999999997. **Our eight-document +request is serial inference, not genuine multi-sequence batching.** It mainly +reduces IPC overhead. These are single comparisons, not throughput distributions. + +Earlier independent media indexing took 668 seconds for 20 assets/118 chunks on +one CPU thread. Frame/recording costs and quality cannot be inferred from fast +text queries. Precise spoken-topic localization, music confusion and no-answer +handling remain failed gates; the earlier experimental media rating is still +4/10. This cache change does not improve that score. +[Actual media evidence](MEDIA-QUALITY.md). + +## Changes made from this audit + +- Added the bounded real-vector cache, with failure/eviction/fingerprint, + filtering/source lifecycle and trace tests. +- Reduced the intentional semantic UI delay from 350 to 120 ms; the separate + 60 ms lexical debounce and bounded request lanes remain. That removes 230 ms + of configured waiting, not a measured 230 ms compositor improvement. +- Kept an existing ready semantic generation searchable during rebuilds. + Initial indexing still needs a generation; activation/warmup still withdraws + backend readiness. Foreground/indexer CPU contention is not solved by this. +- Updated capture validation to use an unseen query when recording actual + worker-inference counters, rather than mislabeling a cache hit. + +## Highest-impact next work + +| Priority | Evidence / proposed work | Required acceptance evidence | +| --- | --- | --- | +| P0 | Bound the whole IPC exchange: synchronous writes currently precede the response timeout. | Nonreading-helper regression; large request deadline and recovery. | +| P0 | Decide the x86 CPU baseline: current static build enables AVX2 without runtime CPU variants. | Declared minimum ISA or tested baseline/AVX2 dispatch; older-CPU launch. | +| P1 | Reuse unchanged document/media vectors; replacement builds currently re-encode all sources. | One-file change embeds only affected chunks; atomic activation and stale/deletion tests. | +| P1 | Pack independent short sequences by token budget; the native worker currently encodes serially. | Vector isolation/parity, length-bucketed CPU/GPU throughput and memory comparisons. | +| P1 | Give interactive queries a shared resource budget/priority during indexing. | First-unseen and warm p95 with real background document/media jobs. | +| P1 | Profile startup stages and compare a smaller query-only context. | Verification/load/graph-warmup/RSS measured separately; no fake first-query cache. | +| P2 | Buffer bounded IPC reads, remove unnecessary stream tying, and make window offset work linear. | Protocol bounds, UTF-8 offsets/tails and real media/payload timing parity. | +| P2 | Consider modality-selective projector loading; current lazy combined projector loads both encoders. | Real text/image/audio RSS and load measurements on supported backends. | + +[DigUp's pinned runtime](https://github.com/ARahim3/DigUp/blob/6faa29e4c315a3a591de1b41bd78751bdf39588d/Sources/LlamaRuntime/LlamaEmbedder.swift) +demonstrates independent-sequence packing and a smaller text-query context. +Those are useful patterns, not proof of an identical speedup on this VM. + +Google supports re-normalized MRL truncation to 512/256/128d. A 256d experiment +could reduce vector storage threefold; it does **not** eliminate the model's +forward pass. Keep 768d until independent retrieval tests pass. Higher vision +budgets trade cost for detail, so retain content-specific quality experiments. +[Google's truncation and modality guidance](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2). + +GPU execution remains untested here. Audit its precision against Google's +BF16/FP32 guidance, and move the inference timing boundary past asynchronous +embedding synchronization before interpreting GPU stage traces. CPU-only results +do not validate accelerator throughput, quality or failure recovery. + +## Reproduce + +The benchmark requires already verified model assets, a compiled CPU worker and +an explicitly selected fixture index. It does not download or index user files. +Read its scope fields and raw samples before comparing hardware or profiles. + +```powershell +python tools/embeddinggemma2/benchmark.py --mode native --models .searchi/models/embeddinggemma2 --worker .searchi/runtime/cpu/Release/searchi-gemma-worker.exe --threads 1 2 4 --repeats 3 --output .searchi/native-performance.json +python tools/embeddinggemma2/benchmark.py --mode mcp --models .searchi/models/embeddinggemma2 --worker .searchi/runtime/cpu/Release/searchi-gemma-worker.exe --cli target/release/searchi.exe --db .searchi/desktop-capture/index.db --repeats 8 --rust-profile release --output .searchi/mcp-performance.json +``` diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 7e902f2..950aaf5 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -96,6 +96,16 @@ real Google inference remains required for product semantic search. Platform, large-corpus, concurrent-indexing and GPU performance gates remain independent. ## Worker round-trip diagnostics +The retained hybrid service now keeps a bounded 32-query vector LRU. Debug +`semantic.query_embedding` spans report `query_cache_hit` and `query_cache_miss`. +A hit reuses an actual model vector and performs no neural inference; it must +not carry the previous worker's tokenization/inference counters. Filters, current +source eligibility, ANN retrieval and fusion still execute. Warmup performs an +actual neural probe and does not seed an unseen foreground-query result. +Compare fresh queries, repeats and filter changes separately. The current +[EmbeddingGemma 2 audit](EMBEDDINGGEMMA2-AUDIT.md) includes paired real-CPU +measurements, startup boundaries and remaining indexing/media costs. + Measured desktop failures return `{message, metrics}` and retain the trace for the failed attempt. Busy retries are separate attempts. If an unexpected worker task failure loses its collector, elapsed time is retained, the trace is null, diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ef7653c..be09ad0 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -80,6 +80,29 @@ Status: implementation started; no production release yet. Windows VM tests use - [ ] Native UI interaction and chosen-provider quality/latency checks before claiming production-ready answers or compatibility with every provider. +## 7. EmbeddingGemma 2 efficiency + +Audit and measured priorities: [EMBEDDINGGEMMA2-AUDIT.md](EMBEDDINGGEMMA2-AUDIT.md). + +- [x] Audit actual required model, task prefixes, pooling/normalization, native + sequence processing and CPU/runtime policies against primary sources. +- [x] Record real CPU 1/2/4-thread query/first-use/document measurements and + paired repeated-query/filter-change results with unchanged retrieval evidence. +- [x] Bounded provider-scoped actual-query vector LRU with accurate cache traces; + preserve filtering, source freshness, failures and actual warmup probes. +- [x] Reduce semantic settle delay to 120 ms and retain old ready generation + during rebuilding; test initial-generation and activation readiness boundaries. +- [ ] Bound whole IPC write/read deadlines and recover from a nonreading helper. +- [ ] Declare/test x86 CPU minimum or baseline/AVX2 runtime selection. +- [ ] Incrementally reuse unchanged document/media vectors in atomic generations. +- [ ] Benchmark genuine token-budgeted multi-sequence batching and isolation. +- [ ] Implement and measure shared foreground/indexer resource budgets/priority. +- [ ] Attribute startup cost; measure smaller query context and selective encoders. +- [ ] Buffered bounded media IPC and linear UTF-8 source-window offset processing. +- [ ] Evaluate 256d MRL and content-specific vision budgets on independent data. +- [ ] Validate GPU precision, synchronization-aware timings, parity and fallback + on actual accelerator hardware; CPU VM checks do not satisfy this gate. + Completion requires recorded evidence; targets are not measured results. ## Completed foundation details diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 8d8f21c..5c43771 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -10,7 +10,7 @@ and `SEARCHI_FORCE_CPU=1`. - Native desktop: **13/13 tests passed**, including structured trace/basic responses and four optional-answer source/consent/revision tests. Post-generation cached query warmup is implemented but its interactive first-query gate remains open. - Frontend: **35/35 tests passed**, TypeScript/Vite production build and + Frontend: **36/36 tests passed**, TypeScript/Vite production build and formatting passed. Preview policy tests cover fractional timestamp seeking, image labels, one active/latest pending request, stale replies and Escape. - Semantic adapter: **8 default tests passed**; the explicit 48 MP resource test @@ -43,8 +43,8 @@ and `SEARCHI_FORCE_CPU=1`. that run: [recorded benchmark](../crates/searchi-core/benchmarks/results/windows-cpu-full-candidate-first-final.json). These measurements do not establish every query or concurrent-indexing goal. -The integrated CPU workspace checkpoint passed **72 default tests** (7 optional AI, -8 assets, 8 CLI/MCP, 24 core, 4 media, 8 semantic, 13 service), with **10 explicit +The integrated CPU workspace checkpoint passed **78 default tests** (7 optional AI, +8 assets, 8 CLI/MCP, 24 core, 4 media, 8 semantic, 19 service), with **10 explicit opt-in tests ignored**. Workspace and native all-target Clippy passed with warnings denied. The later semantic CPU-restart counter change passed workspace Clippy and the actual CPU query diagnostics test. Earlier aggregate counts are historical. @@ -128,6 +128,18 @@ agent execution and release packaging remain open gates. ## Historical evidence +The [EmbeddingGemma 2 audit](EMBEDDINGGEMMA2-AUDIT.md) records new CPU diagnostics: +24 uncached native queries per 1/2/4-thread setting, paired real MCP repeat/filter +measurements, 20 unchanged result-set comparisons, actual Release startup/first +query, and document single-versus-serial-batch parity. Six cache-control tests +verify eviction, fingerprint/trim behavior, failures, unchanged filtering/source +lifecycle and accurate hit traces. The frontend readiness regression retains an +old committed generation during rebuilds and blocks initial/activation states. +The four real-engine browser captures were regenerated and visually inspected; +the performance capture deliberately uses an unseen query and proves actual +worker inference rather than treating a cached vector as a new neural pass. +Native visual/paint and production media relevance gates remain open. + Initial engine/CLI checks covered retrieval, persistence, filesystem changes, exclusions/filters, migration, concurrent readers, PDF pages, DOCX paragraphs, corrupt replacement, encrypted PDF errors and bounded compressed input. diff --git a/docs/evidence/embeddinggemma2-mcp-after.json b/docs/evidence/embeddinggemma2-mcp-after.json new file mode 100644 index 0000000..6cb665b --- /dev/null +++ b/docs/evidence/embeddinggemma2-mcp-after.json @@ -0,0 +1,730 @@ +{ + "captured_at_utc": "2026-10-12T18:41:00Z", + "os": "Windows-11-10.0.26300-SP0", + "machine": "AMD64", + "logical_cpus": 4, + "cpu_only": true, + "rust_profile": "debug", + "label": "bounded-query-vector-cache", + "worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3", + "cli_sha256": "4a4c69c34f48162e2033fbfbc7b3500aa7496f6c1d9a03be6d611cf84bafe6e0", + "results": { + "startup_to_initialized_ms": 11040.071900002658, + "stats": { + "files": { + "content_files": 5, + "files": 7 + }, + "mode": "hybrid", + "semantic_available": true, + "worker": { + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "runtime": "b11461-no-logits-v1", + "threads": 1 + } + }, + "samples": [ + { + "case": "first_unseen", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 79.04949999647215, + "engine_ms": 77.1732, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.6412000006530434, + "engine_ms": 1.4406, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.3130000045057386, + "engine_ms": 1.1769, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.4286000116262585, + "engine_ms": 1.2572, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.217100001871586, + "engine_ms": 1.0070000000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.3689999943599105, + "engine_ms": 1.0303, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.5655000063125044, + "engine_ms": 1.1261, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.9205000028014183, + "engine_ms": 1.395, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 2.2301000135485083, + "engine_ms": 1.1747999999999998, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "pdf", + "request_ms": 1.8029999919235706, + "engine_ms": 0.975, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "md", + "request_ms": 2.3480000090785325, + "engine_ms": 1.0859, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "pdf", + "request_ms": 2.0995999802835286, + "engine_ms": 1.0337, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "md", + "request_ms": 2.343600004678592, + "engine_ms": 1.1913, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "mountains reflected in calm water", + "extension": null, + "request_ms": 75.07760002044961, + "engine_ms": 73.8136, + "matches": [ + { + "filename": "Mountain Lake.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.01639344262295082 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "die Kündigungsfrist im Lieferantenvertrag", + "extension": null, + "request_ms": 89.54819999053143, + "engine_ms": 86.77550000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "source code that checks indexed file permissions", + "extension": null, + "request_ms": 77.97480002045631, + "engine_ms": 76.4551, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "music with piano and a slow melody", + "extension": null, + "request_ms": 78.43809999758378, + "engine_ms": 76.8372, + "matches": [ + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.01639344262295082 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "a boat crossing a lake at sunset", + "extension": null, + "request_ms": 77.43000000482425, + "engine_ms": 75.7792, + "matches": [ + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.01639344262295082 + }, + { + "filename": "Mountain Lake.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "notes about returning unused materials", + "extension": null, + "request_ms": 70.03259999328293, + "engine_ms": 68.49170000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "find the recording discussing orange soil", + "extension": null, + "request_ms": 78.86860001599416, + "engine_ms": 77.1636, + "matches": [ + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.015873015873015872 + } + ] + } + ], + "same_query": { + "samples": 8, + "mean_ms": 1.2009875, + "p50_ms": 1.17585, + "p95_ms": 1.4406 + }, + "filter_change": { + "samples": 4, + "mean_ms": 1.071475, + "p50_ms": 1.0598, + "p95_ms": 1.1913 + }, + "scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint." + } +} diff --git a/docs/evidence/embeddinggemma2-mcp-before.json b/docs/evidence/embeddinggemma2-mcp-before.json new file mode 100644 index 0000000..92113e3 --- /dev/null +++ b/docs/evidence/embeddinggemma2-mcp-before.json @@ -0,0 +1,732 @@ +{ + "captured_at_utc": "2026-10-12T18:35:22Z", + "os": "Windows-11-10.0.26300-SP0", + "machine": "AMD64", + "logical_cpus": 4, + "cpu_only": true, + "rust_profile": "debug", + "results": { + "startup_to_initialized_ms": 7216.115900024306, + "stats": { + "files": { + "content_files": 5, + "files": 7 + }, + "mode": "hybrid", + "semantic_available": true, + "worker": { + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "runtime": "b11461-no-logits-v1", + "threads": 1 + } + }, + "samples": [ + { + "case": "first_unseen", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 85.7454999932088, + "engine_ms": 82.9858, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 84.06170000671409, + "engine_ms": 82.0945, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 69.04040000517853, + "engine_ms": 67.36319999999999, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 66.24039998860098, + "engine_ms": 64.4344, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 67.91299997712485, + "engine_ms": 66.0718, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 70.40650001727045, + "engine_ms": 68.77619999999999, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 79.71600000746548, + "engine_ms": 77.87230000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 69.87529998878017, + "engine_ms": 68.0868, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 77.02430000063032, + "engine_ms": 74.836, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "pdf", + "request_ms": 84.02040001237765, + "engine_ms": 79.5709, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "md", + "request_ms": 72.45289999991655, + "engine_ms": 70.70519999999999, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "pdf", + "request_ms": 68.47949998336844, + "engine_ms": 66.9529, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "md", + "request_ms": 68.12429998535663, + "engine_ms": 66.5463, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "mountains reflected in calm water", + "extension": null, + "request_ms": 74.91510000545532, + "engine_ms": 73.53999999999999, + "matches": [ + { + "filename": "Mountain Lake.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.01639344262295082 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "die Kündigungsfrist im Lieferantenvertrag", + "extension": null, + "request_ms": 84.24939998076297, + "engine_ms": 82.6408, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "source code that checks indexed file permissions", + "extension": null, + "request_ms": 79.35359998373315, + "engine_ms": 77.66290000000001, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "music with piano and a slow melody", + "extension": null, + "request_ms": 80.26409999001771, + "engine_ms": 78.6324, + "matches": [ + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.01639344262295082 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "a boat crossing a lake at sunset", + "extension": null, + "request_ms": 79.87940000020899, + "engine_ms": 78.0203, + "matches": [ + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.01639344262295082 + }, + { + "filename": "Mountain Lake.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "notes about returning unused materials", + "extension": null, + "request_ms": 73.2068000070285, + "engine_ms": 71.48490000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "find the recording discussing orange soil", + "extension": null, + "request_ms": 79.31890001054853, + "engine_ms": 77.6251, + "matches": [ + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.015873015873015872 + } + ] + } + ], + "same_query": { + "samples": 8, + "mean_ms": 71.1919, + "p50_ms": 68.4315, + "p95_ms": 82.0945 + }, + "filter_change": { + "samples": 4, + "mean_ms": 70.943825, + "p50_ms": 68.82905, + "p95_ms": 79.5709 + }, + "scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint." + }, + "label": "baseline-no-query-vector-cache", + "worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3", + "cli_sha256": null, + "measurement_base_commit": "9870df7", + "metadata_note": "Worker unchanged during these runs; hash recorded afterward. Baseline CLI executable hash was not captured; source checkpoint was 9870df7." +} diff --git a/docs/evidence/embeddinggemma2-mcp-release.json b/docs/evidence/embeddinggemma2-mcp-release.json new file mode 100644 index 0000000..0cd9a12 --- /dev/null +++ b/docs/evidence/embeddinggemma2-mcp-release.json @@ -0,0 +1,730 @@ +{ + "captured_at_utc": "2026-10-12T18:47:54Z", + "os": "Windows-11-10.0.26300-SP0", + "machine": "AMD64", + "logical_cpus": 4, + "cpu_only": true, + "rust_profile": "release", + "label": "release-query-vector-cache", + "worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3", + "cli_sha256": "9e6c087c863b606d02534bff2db1eb397df3a2b8c9aa2f875b2e5ac0a2d04316", + "results": { + "startup_to_initialized_ms": 4878.2795000006445, + "stats": { + "files": { + "content_files": 5, + "files": 7 + }, + "mode": "hybrid", + "semantic_available": true, + "worker": { + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "runtime": "b11461-no-logits-v1", + "threads": 1 + } + }, + "samples": [ + { + "case": "first_unseen", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 82.9809999850113, + "engine_ms": 81.94659999999999, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.6709000046830624, + "engine_ms": 0.777, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.6380999877583236, + "engine_ms": 0.9057, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.4798000047449023, + "engine_ms": 0.6547000000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.6269000188913196, + "engine_ms": 0.8484999999999999, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.3796999992337078, + "engine_ms": 0.6398, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.5811999910511076, + "engine_ms": 0.8515, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.2971000105608255, + "engine_ms": 0.6551, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "same_query", + "query": "cancel the supplier partnership", + "extension": null, + "request_ms": 1.2206999817863107, + "engine_ms": 0.7019, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "pdf", + "request_ms": 1.7168000049423426, + "engine_ms": 0.6491, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "md", + "request_ms": 1.3719999988097697, + "engine_ms": 0.6258, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "pdf", + "request_ms": 1.505599997472018, + "engine_ms": 0.6131, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + } + ] + }, + { + "case": "filter_change", + "query": "cancel the supplier partnership", + "extension": "md", + "request_ms": 1.6737999976612628, + "engine_ms": 0.9121, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "mountains reflected in calm water", + "extension": null, + "request_ms": 77.39190000575036, + "engine_ms": 75.18950000000001, + "matches": [ + { + "filename": "Mountain Lake.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.01639344262295082 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "die Kündigungsfrist im Lieferantenvertrag", + "extension": null, + "request_ms": 98.24019999359734, + "engine_ms": 97.0811, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "source code that checks indexed file permissions", + "extension": null, + "request_ms": 91.89109998987988, + "engine_ms": 90.8722, + "matches": [ + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "music with piano and a slow melody", + "extension": null, + "request_ms": 85.49989998573437, + "engine_ms": 84.59949999999999, + "matches": [ + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.01639344262295082 + }, + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "a boat crossing a lake at sunset", + "extension": null, + "request_ms": 86.64829999906942, + "engine_ms": 85.5861, + "matches": [ + { + "filename": "Night Pier.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.01639344262295082 + }, + { + "filename": "Mountain Lake.jpg", + "source": { + "kind": "image", + "number": 0 + }, + "snippet": "Image content", + "score": 0.016129032258064516 + }, + { + "filename": "Travel Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "notes about returning unused materials", + "extension": null, + "request_ms": 69.34839999303222, + "engine_ms": 68.37740000000001, + "matches": [ + { + "filename": "Supplier Agreement.pdf", + "source": { + "kind": "page", + "number": 2 + }, + "snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n", + "score": 0.01639344262295082 + }, + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.016129032258064516 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.015873015873015872 + } + ] + }, + { + "case": "distinct_query", + "query": "find the recording discussing orange soil", + "extension": null, + "request_ms": 79.9731999868527, + "engine_ms": 78.8852, + "matches": [ + { + "filename": "Garden Notes.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.", + "score": 0.01639344262295082 + }, + { + "filename": "Q3 Procurement.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.", + "score": 0.016129032258064516 + }, + { + "filename": "Vendor Onboarding.md", + "source": { + "kind": "line", + "number": 1 + }, + "snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.", + "score": 0.015873015873015872 + } + ] + } + ], + "same_query": { + "samples": 8, + "mean_ms": 0.754275, + "p50_ms": 0.7394499999999999, + "p95_ms": 0.9057 + }, + "filter_change": { + "samples": 4, + "mean_ms": 0.700025, + "p50_ms": 0.6374500000000001, + "p95_ms": 0.9121 + }, + "scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint." + } +} diff --git a/docs/evidence/embeddinggemma2-native-cpu.json b/docs/evidence/embeddinggemma2-native-cpu.json new file mode 100644 index 0000000..79b97e0 --- /dev/null +++ b/docs/evidence/embeddinggemma2-native-cpu.json @@ -0,0 +1,1066 @@ +{ + "captured_at_utc": "2026-10-12T18:36:02Z", + "os": "Windows-11-10.0.26300-SP0", + "machine": "AMD64", + "logical_cpus": 4, + "cpu_only": true, + "rust_profile": "not-applicable", + "results": { + "text_model_verification_ms": 169.60700001800433, + "thread_reports": [ + { + "threads": 1, + "worker_info": { + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "runtime": "b11461-no-logits-v1", + "threads": 1 + }, + "process_load_ms": 815.7074000046123, + "first_query_ms": 113.29499998828396, + "first_query_worker": { + "batch_prepare_us": 16, + "inference_us": 112214, + "pool_normalize_us": 15, + "token_count": 13, + "tokenization_us": 129 + }, + "warm_queries": { + "samples": 24, + "mean_ms": 75.851649997882, + "p50_ms": 74.90404999407474, + "p95_ms": 86.58900001319125 + }, + "query_samples": [ + { + "query": "cancel the supplier partnership", + "repeat": 0, + "round_trip_ms": 73.44800001010299, + "worker": { + "batch_prepare_us": 15, + "inference_us": 72303, + "pool_normalize_us": 15, + "token_count": 13, + "tokenization_us": 137 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 0, + "round_trip_ms": 74.44210001267493, + "worker": { + "batch_prepare_us": 19, + "inference_us": 73371, + "pool_normalize_us": 15, + "token_count": 14, + "tokenization_us": 150 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 0, + "round_trip_ms": 86.18839998962358, + "worker": { + "batch_prepare_us": 17, + "inference_us": 85159, + "pool_normalize_us": 16, + "token_count": 18, + "tokenization_us": 155 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 0, + "round_trip_ms": 79.52499997918494, + "worker": { + "batch_prepare_us": 18, + "inference_us": 78336, + "pool_normalize_us": 16, + "token_count": 16, + "tokenization_us": 301 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 0, + "round_trip_ms": 86.58900001319125, + "worker": { + "batch_prepare_us": 17, + "inference_us": 85332, + "pool_normalize_us": 20, + "token_count": 16, + "tokenization_us": 135 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 0, + "round_trip_ms": 81.7858999944292, + "worker": { + "batch_prepare_us": 17, + "inference_us": 80799, + "pool_normalize_us": 18, + "token_count": 16, + "tokenization_us": 140 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 0, + "round_trip_ms": 73.54330000816844, + "worker": { + "batch_prepare_us": 17, + "inference_us": 72488, + "pool_normalize_us": 15, + "token_count": 14, + "tokenization_us": 158 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 0, + "round_trip_ms": 73.09299998451024, + "worker": { + "batch_prepare_us": 22, + "inference_us": 71638, + "pool_normalize_us": 16, + "token_count": 15, + "tokenization_us": 280 + } + }, + { + "query": "cancel the supplier partnership", + "repeat": 1, + "round_trip_ms": 64.92720000096597, + "worker": { + "batch_prepare_us": 21, + "inference_us": 63683, + "pool_normalize_us": 14, + "token_count": 13, + "tokenization_us": 141 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 1, + "round_trip_ms": 71.13940000999719, + "worker": { + "batch_prepare_us": 20, + "inference_us": 69811, + "pool_normalize_us": 14, + "token_count": 14, + "tokenization_us": 137 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 1, + "round_trip_ms": 86.62129999720491, + "worker": { + "batch_prepare_us": 17, + "inference_us": 85580, + "pool_normalize_us": 15, + "token_count": 18, + "tokenization_us": 166 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 1, + "round_trip_ms": 74.0920000243932, + "worker": { + "batch_prepare_us": 16, + "inference_us": 72999, + "pool_normalize_us": 15, + "token_count": 16, + "tokenization_us": 164 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 1, + "round_trip_ms": 73.72109999414533, + "worker": { + "batch_prepare_us": 20, + "inference_us": 72644, + "pool_normalize_us": 14, + "token_count": 16, + "tokenization_us": 143 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 1, + "round_trip_ms": 84.5557999855373, + "worker": { + "batch_prepare_us": 16, + "inference_us": 83456, + "pool_normalize_us": 14, + "token_count": 16, + "tokenization_us": 129 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 1, + "round_trip_ms": 78.47320000291802, + "worker": { + "batch_prepare_us": 18, + "inference_us": 77300, + "pool_normalize_us": 23, + "token_count": 14, + "tokenization_us": 146 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 1, + "round_trip_ms": 72.73319998057559, + "worker": { + "batch_prepare_us": 16, + "inference_us": 71599, + "pool_normalize_us": 15, + "token_count": 15, + "tokenization_us": 239 + } + }, + { + "query": "cancel the supplier partnership", + "repeat": 2, + "round_trip_ms": 65.36449998384342, + "worker": { + "batch_prepare_us": 15, + "inference_us": 64347, + "pool_normalize_us": 14, + "token_count": 13, + "tokenization_us": 139 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 2, + "round_trip_ms": 66.73039999441244, + "worker": { + "batch_prepare_us": 15, + "inference_us": 65666, + "pool_normalize_us": 15, + "token_count": 14, + "tokenization_us": 138 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 2, + "round_trip_ms": 79.93279999936931, + "worker": { + "batch_prepare_us": 17, + "inference_us": 78666, + "pool_normalize_us": 14, + "token_count": 18, + "tokenization_us": 157 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 2, + "round_trip_ms": 76.67539999238215, + "worker": { + "batch_prepare_us": 16, + "inference_us": 75632, + "pool_normalize_us": 15, + "token_count": 16, + "tokenization_us": 159 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 2, + "round_trip_ms": 75.9803999972064, + "worker": { + "batch_prepare_us": 17, + "inference_us": 74668, + "pool_normalize_us": 15, + "token_count": 16, + "tokenization_us": 251 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 2, + "round_trip_ms": 77.74100001552142, + "worker": { + "batch_prepare_us": 17, + "inference_us": 76779, + "pool_normalize_us": 15, + "token_count": 16, + "tokenization_us": 133 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 2, + "round_trip_ms": 67.77120000333525, + "worker": { + "batch_prepare_us": 18, + "inference_us": 66686, + "pool_normalize_us": 16, + "token_count": 14, + "tokenization_us": 148 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 2, + "round_trip_ms": 75.36599997547455, + "worker": { + "batch_prepare_us": 16, + "inference_us": 74247, + "pool_normalize_us": 16, + "token_count": 15, + "tokenization_us": 162 + } + } + ], + "window_samples": [ + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 43.25290000997484 + }, + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 41.934500011848286 + }, + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 41.633900022134185 + } + ], + "documents": { + "single_request_ms": [ + 445.60949999140576, + 455.59680002043024, + 460.02140000928193, + 468.72129998519085, + 553.4862999920733, + 514.9270999827422, + 496.3241000077687, + 440.382500004489 + ], + "eight_single_total_ms": 3835.068999993382, + "eight_document_request_ms": 3635.880999994697, + "batch_cosine_min": 0.9999999969587648 + } + }, + { + "threads": 2, + "worker_info": { + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "runtime": "b11461-no-logits-v1", + "threads": 2 + }, + "process_load_ms": 653.204200003529, + "first_query_ms": 64.24510001670569, + "first_query_worker": { + "batch_prepare_us": 16, + "inference_us": 62932, + "pool_normalize_us": 13, + "token_count": 13, + "tokenization_us": 138 + }, + "warm_queries": { + "samples": 24, + "mean_ms": 41.18115833504513, + "p50_ms": 40.798300004098564, + "p95_ms": 46.66200000792742 + }, + "query_samples": [ + { + "query": "cancel the supplier partnership", + "repeat": 0, + "round_trip_ms": 41.949400008888915, + "worker": { + "batch_prepare_us": 19, + "inference_us": 40943, + "pool_normalize_us": 13, + "token_count": 13, + "tokenization_us": 217 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 0, + "round_trip_ms": 42.03760001109913, + "worker": { + "batch_prepare_us": 20, + "inference_us": 41026, + "pool_normalize_us": 17, + "token_count": 14, + "tokenization_us": 140 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 0, + "round_trip_ms": 49.49269999633543, + "worker": { + "batch_prepare_us": 16, + "inference_us": 48450, + "pool_normalize_us": 12, + "token_count": 18, + "tokenization_us": 150 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 0, + "round_trip_ms": 43.35610000998713, + "worker": { + "batch_prepare_us": 16, + "inference_us": 42259, + "pool_normalize_us": 14, + "token_count": 16, + "tokenization_us": 153 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 0, + "round_trip_ms": 45.92119998415001, + "worker": { + "batch_prepare_us": 16, + "inference_us": 44979, + "pool_normalize_us": 16, + "token_count": 16, + "tokenization_us": 136 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 0, + "round_trip_ms": 46.36209999443963, + "worker": { + "batch_prepare_us": 17, + "inference_us": 45314, + "pool_normalize_us": 18, + "token_count": 16, + "tokenization_us": 133 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 0, + "round_trip_ms": 37.06100000999868, + "worker": { + "batch_prepare_us": 16, + "inference_us": 36142, + "pool_normalize_us": 12, + "token_count": 14, + "tokenization_us": 151 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 0, + "round_trip_ms": 39.39550000359304, + "worker": { + "batch_prepare_us": 15, + "inference_us": 38445, + "pool_normalize_us": 12, + "token_count": 15, + "tokenization_us": 147 + } + }, + { + "query": "cancel the supplier partnership", + "repeat": 1, + "round_trip_ms": 35.09940000367351, + "worker": { + "batch_prepare_us": 15, + "inference_us": 34104, + "pool_normalize_us": 13, + "token_count": 13, + "tokenization_us": 137 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 1, + "round_trip_ms": 39.29329998209141, + "worker": { + "batch_prepare_us": 14, + "inference_us": 38389, + "pool_normalize_us": 13, + "token_count": 14, + "tokenization_us": 132 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 1, + "round_trip_ms": 45.58560001896694, + "worker": { + "batch_prepare_us": 16, + "inference_us": 44574, + "pool_normalize_us": 13, + "token_count": 18, + "tokenization_us": 153 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 1, + "round_trip_ms": 41.35210000094958, + "worker": { + "batch_prepare_us": 16, + "inference_us": 40337, + "pool_normalize_us": 17, + "token_count": 16, + "tokenization_us": 264 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 1, + "round_trip_ms": 41.10010000295006, + "worker": { + "batch_prepare_us": 12, + "inference_us": 40112, + "pool_normalize_us": 12, + "token_count": 16, + "tokenization_us": 130 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 1, + "round_trip_ms": 42.93260001577437, + "worker": { + "batch_prepare_us": 18, + "inference_us": 42017, + "pool_normalize_us": 12, + "token_count": 16, + "tokenization_us": 123 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 1, + "round_trip_ms": 36.097200005315244, + "worker": { + "batch_prepare_us": 18, + "inference_us": 35212, + "pool_normalize_us": 13, + "token_count": 14, + "tokenization_us": 144 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 1, + "round_trip_ms": 39.3630999897141, + "worker": { + "batch_prepare_us": 14, + "inference_us": 38421, + "pool_normalize_us": 13, + "token_count": 15, + "tokenization_us": 140 + } + }, + { + "query": "cancel the supplier partnership", + "repeat": 2, + "round_trip_ms": 35.48319998662919, + "worker": { + "batch_prepare_us": 15, + "inference_us": 34345, + "pool_normalize_us": 13, + "token_count": 13, + "tokenization_us": 142 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 2, + "round_trip_ms": 36.904399981722236, + "worker": { + "batch_prepare_us": 14, + "inference_us": 35889, + "pool_normalize_us": 13, + "token_count": 14, + "tokenization_us": 130 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 2, + "round_trip_ms": 46.66200000792742, + "worker": { + "batch_prepare_us": 15, + "inference_us": 45678, + "pool_normalize_us": 12, + "token_count": 18, + "tokenization_us": 148 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 2, + "round_trip_ms": 46.57570002018474, + "worker": { + "batch_prepare_us": 17, + "inference_us": 45620, + "pool_normalize_us": 13, + "token_count": 16, + "tokenization_us": 156 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 2, + "round_trip_ms": 38.74439999344759, + "worker": { + "batch_prepare_us": 17, + "inference_us": 37779, + "pool_normalize_us": 12, + "token_count": 16, + "tokenization_us": 137 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 2, + "round_trip_ms": 40.49650000524707, + "worker": { + "batch_prepare_us": 21, + "inference_us": 39346, + "pool_normalize_us": 17, + "token_count": 16, + "tokenization_us": 172 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 2, + "round_trip_ms": 37.55730000557378, + "worker": { + "batch_prepare_us": 15, + "inference_us": 36487, + "pool_normalize_us": 14, + "token_count": 14, + "tokenization_us": 142 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 2, + "round_trip_ms": 39.525300002424046, + "worker": { + "batch_prepare_us": 15, + "inference_us": 38386, + "pool_normalize_us": 12, + "token_count": 15, + "tokenization_us": 146 + } + } + ], + "window_samples": [ + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 47.179399989545345 + }, + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 43.76179998507723 + }, + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 41.03860000032 + } + ], + "documents": { + "single_request_ms": [ + 236.73520001466386, + 256.55920000281185, + 239.2048000183422, + 227.28650001226924, + 225.1213999988977, + 247.02910002088174, + 233.4864000149537, + 225.114200002281 + ], + "eight_single_total_ms": 1890.5368000851013, + "eight_document_request_ms": 1870.5574000196066, + "batch_cosine_min": 0.9999999969587648 + } + }, + { + "threads": 4, + "worker_info": { + "device": "cpu", + "dimensions": 768, + "media_supported": true, + "model": "google/embeddinggemma-2", + "ready": true, + "runtime": "b11461-no-logits-v1", + "threads": 4 + }, + "process_load_ms": 663.6421000002883, + "first_query_ms": 49.979600007645786, + "first_query_worker": { + "batch_prepare_us": 24, + "inference_us": 48969, + "pool_normalize_us": 14, + "token_count": 13, + "tokenization_us": 165 + }, + "warm_queries": { + "samples": 24, + "mean_ms": 35.79947917023674, + "p50_ms": 34.164500000770204, + "p95_ms": 49.859899998409674 + }, + "query_samples": [ + { + "query": "cancel the supplier partnership", + "repeat": 0, + "round_trip_ms": 36.131300003034994, + "worker": { + "batch_prepare_us": 17, + "inference_us": 35103, + "pool_normalize_us": 21, + "token_count": 13, + "tokenization_us": 134 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 0, + "round_trip_ms": 34.27959998953156, + "worker": { + "batch_prepare_us": 30, + "inference_us": 33008, + "pool_normalize_us": 13, + "token_count": 14, + "tokenization_us": 166 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 0, + "round_trip_ms": 39.186100009828806, + "worker": { + "batch_prepare_us": 16, + "inference_us": 38221, + "pool_normalize_us": 15, + "token_count": 18, + "tokenization_us": 165 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 0, + "round_trip_ms": 34.25729999435134, + "worker": { + "batch_prepare_us": 16, + "inference_us": 33191, + "pool_normalize_us": 15, + "token_count": 16, + "tokenization_us": 158 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 0, + "round_trip_ms": 30.50790002453141, + "worker": { + "batch_prepare_us": 14, + "inference_us": 29711, + "pool_normalize_us": 12, + "token_count": 16, + "tokenization_us": 127 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 0, + "round_trip_ms": 33.78009999869391, + "worker": { + "batch_prepare_us": 15, + "inference_us": 32892, + "pool_normalize_us": 12, + "token_count": 16, + "tokenization_us": 129 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 0, + "round_trip_ms": 30.399200011743233, + "worker": { + "batch_prepare_us": 12, + "inference_us": 29189, + "pool_normalize_us": 14, + "token_count": 14, + "tokenization_us": 238 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 0, + "round_trip_ms": 49.859899998409674, + "worker": { + "batch_prepare_us": 15, + "inference_us": 48908, + "pool_normalize_us": 14, + "token_count": 15, + "tokenization_us": 152 + } + }, + { + "query": "cancel the supplier partnership", + "repeat": 1, + "round_trip_ms": 35.43660001014359, + "worker": { + "batch_prepare_us": 16, + "inference_us": 34386, + "pool_normalize_us": 13, + "token_count": 13, + "tokenization_us": 142 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 1, + "round_trip_ms": 34.17490000720136, + "worker": { + "batch_prepare_us": 14, + "inference_us": 32944, + "pool_normalize_us": 15, + "token_count": 14, + "tokenization_us": 241 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 1, + "round_trip_ms": 57.266200019512326, + "worker": { + "batch_prepare_us": 16, + "inference_us": 50080, + "pool_normalize_us": 16, + "token_count": 18, + "tokenization_us": 153 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 1, + "round_trip_ms": 48.428500012960285, + "worker": { + "batch_prepare_us": 23, + "inference_us": 47407, + "pool_normalize_us": 13, + "token_count": 16, + "tokenization_us": 184 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 1, + "round_trip_ms": 39.79020001133904, + "worker": { + "batch_prepare_us": 14, + "inference_us": 38782, + "pool_normalize_us": 15, + "token_count": 16, + "tokenization_us": 141 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 1, + "round_trip_ms": 40.077999990899116, + "worker": { + "batch_prepare_us": 17, + "inference_us": 38731, + "pool_normalize_us": 277, + "token_count": 16, + "tokenization_us": 134 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 1, + "round_trip_ms": 31.26170000177808, + "worker": { + "batch_prepare_us": 16, + "inference_us": 30124, + "pool_normalize_us": 12, + "token_count": 14, + "tokenization_us": 153 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 1, + "round_trip_ms": 31.994100019801408, + "worker": { + "batch_prepare_us": 15, + "inference_us": 31071, + "pool_normalize_us": 12, + "token_count": 15, + "tokenization_us": 147 + } + }, + { + "query": "cancel the supplier partnership", + "repeat": 2, + "round_trip_ms": 34.15409999433905, + "worker": { + "batch_prepare_us": 14, + "inference_us": 33002, + "pool_normalize_us": 17, + "token_count": 13, + "tokenization_us": 130 + } + }, + { + "query": "mountains reflected in calm water", + "repeat": 2, + "round_trip_ms": 29.930799995781854, + "worker": { + "batch_prepare_us": 12, + "inference_us": 28386, + "pool_normalize_us": 12, + "token_count": 14, + "tokenization_us": 133 + } + }, + { + "query": "die Kündigungsfrist im Lieferantenvertrag", + "repeat": 2, + "round_trip_ms": 31.65729998727329, + "worker": { + "batch_prepare_us": 22, + "inference_us": 30362, + "pool_normalize_us": 13, + "token_count": 18, + "tokenization_us": 146 + } + }, + { + "query": "source code that checks indexed file permissions", + "repeat": 2, + "round_trip_ms": 45.445800002198666, + "worker": { + "batch_prepare_us": 22, + "inference_us": 44573, + "pool_normalize_us": 12, + "token_count": 16, + "tokenization_us": 161 + } + }, + { + "query": "music with piano and a slow melody", + "repeat": 2, + "round_trip_ms": 26.717100001405925, + "worker": { + "batch_prepare_us": 17, + "inference_us": 25810, + "pool_normalize_us": 14, + "token_count": 16, + "tokenization_us": 132 + } + }, + { + "query": "a boat crossing a lake at sunset", + "repeat": 2, + "round_trip_ms": 29.474100010702386, + "worker": { + "batch_prepare_us": 13, + "inference_us": 28671, + "pool_normalize_us": 17, + "token_count": 16, + "tokenization_us": 128 + } + }, + { + "query": "notes about returning unused materials", + "repeat": 2, + "round_trip_ms": 28.210099990246817, + "worker": { + "batch_prepare_us": 14, + "inference_us": 27405, + "pool_normalize_us": 14, + "token_count": 14, + "tokenization_us": 140 + } + }, + { + "query": "find the recording discussing orange soil", + "repeat": 2, + "round_trip_ms": 26.76659999997355, + "worker": { + "batch_prepare_us": 14, + "inference_us": 25973, + "pool_normalize_us": 12, + "token_count": 15, + "tokenization_us": 140 + } + } + ], + "window_samples": [ + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 44.72370000439696 + }, + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 42.06910001812503 + }, + { + "bytes": 50000, + "windows": 5, + "round_trip_ms": 43.17590000573546 + } + ], + "documents": { + "single_request_ms": [ + 168.01689998828806, + 177.8370999963954, + 143.9599999866914, + 143.07769999140874, + 158.93220002180897, + 165.58800000348128, + 140.45990002341568, + 142.65200000954792 + ], + "eight_single_total_ms": 1240.5238000210375, + "eight_document_request_ms": 1241.758900025161, + "batch_cosine_min": 0.9999999969587648 + } + } + ], + "scope": "Native actual Google Q8 worker; verified model; cached OS pages possible. Includes NDJSON encode/decode; excludes Rust adapter, ANN and UI. First query precedes warm samples. Documents measure IPC batch, not assumed parallelism." + }, + "label": "native-worker-before-query-cache", + "worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3", + "cli_sha256": null, + "measurement_base_commit": "9870df7", + "metadata_note": "Worker unchanged during these runs; hash recorded afterward. Baseline CLI executable hash was not captured; source checkpoint was 9870df7." +} diff --git a/docs/screenshots/capture-manifest.json b/docs/screenshots/capture-manifest.json index 7371e14..80d0107 100644 --- a/docs/screenshots/capture-manifest.json +++ b/docs/screenshots/capture-manifest.json @@ -1,7 +1,7 @@ { "kind": "browser-rendered-production-ui-real-cpu-engine", "source": { - "base_commit": "b8400c4fbf4038820c383ed429207a3b339bf0a5", + "base_commit": "9870df7cb5b537ba516997a0d049b996bc30b08d", "working_tree_dirty": true }, "viewport": { @@ -37,7 +37,7 @@ "metrics": { "dropped_spans": 0, "result_count": 7, - "total_ms": 84.7475, + "total_ms": 2.0688999999999997, "trace": null }, "first_filename": "Supplier Agreement.pdf", @@ -58,7 +58,7 @@ "metrics": { "dropped_spans": 0, "result_count": 7, - "total_ms": 83.50019999999999, + "total_ms": 1.5953, "trace": null }, "first_filename": "Mountain Lake.jpg", @@ -71,183 +71,185 @@ } }, { - "query": "cancel the supplier partnership", + "query": "how can we cancel the supplier partnership", "mode": "debug", "result_count": 7, "metrics": { "dropped_spans": 0, "result_count": 7, - "total_ms": 81.5728, + "total_ms": 92.8279, "trace": [ { "counters": {}, - "duration_ms": 81.5544, + "duration_ms": 92.80539999999999, "name": "hybrid.search", "parent": null, "start_ms": 0.0106 }, { "counters": {}, - "duration_ms": 1.5203, + "duration_ms": 2.6003, "name": "lexical.search", "parent": 0, - "start_ms": 0.0281 + "start_ms": 0.027 }, { "counters": { - "query_bytes": 31 + "query_bytes": 42 }, - "duration_ms": 0.0383, + "duration_ms": 0.0389, "name": "lexical.normalization", "parent": 1, - "start_ms": 0.0386 + "start_ms": 0.0374 }, { "counters": { - "frequency_cache_hits": 29, - "frequency_cache_misses": 0, + "frequency_cache_hits": 30, + "frequency_cache_misses": 8, "trigram_index": 1 }, - "duration_ms": 0.1389, + "duration_ms": 1.0081, "name": "lexical.candidate_plan", "parent": 1, - "start_ms": 0.0871 + "start_ms": 0.08650000000000001 }, { "counters": {}, - "duration_ms": 0.1952, + "duration_ms": 0.2286, "name": "lexical.exact_filename", "parent": 1, - "start_ms": 0.2358 + "start_ms": 1.1168 }, { "counters": { "filters_active": 0, "rows": 0 }, - "duration_ms": 0.16620000000000001, + "duration_ms": 0.1837, "name": "lexical.filtered_sql", "parent": 4, - "start_ms": 0.254 + "start_ms": 1.1471 }, { "counters": {}, - "duration_ms": 0.183, + "duration_ms": 0.2688, "name": "lexical.filename_substring", "parent": 1, - "start_ms": 0.44539999999999996 + "start_ms": 1.3663 }, { "counters": { "filters_active": 0, "rows": 0 }, - "duration_ms": 0.15589999999999998, + "duration_ms": 0.2249, "name": "lexical.filtered_sql", "parent": 6, - "start_ms": 0.4609 + "start_ms": 1.3906999999999998 }, { "counters": {}, - "duration_ms": 0.1605, + "duration_ms": 0.2146, "name": "lexical.path_substring", "parent": 1, - "start_ms": 0.642 + "start_ms": 1.6556 }, { "counters": { "filters_active": 0, "rows": 0 }, - "duration_ms": 0.1332, + "duration_ms": 0.175, "name": "lexical.filtered_sql", "parent": 8, - "start_ms": 0.6575 + "start_ms": 1.6802 }, { "counters": {}, - "duration_ms": 0.1908, + "duration_ms": 0.2391, "name": "lexical.content", "parent": 1, - "start_ms": 0.8159000000000001 + "start_ms": 1.8861999999999999 }, { "counters": { "filters_active": 0, "rows": 0 }, - "duration_ms": 0.1622, + "duration_ms": 0.2053, "name": "lexical.filtered_sql", "parent": 10, - "start_ms": 0.8313 + "start_ms": 1.9049 }, { "counters": {}, - "duration_ms": 0.21509999999999999, + "duration_ms": 0.3054, "name": "lexical.filename_terms", "parent": 1, - "start_ms": 1.0205 + "start_ms": 2.1413 }, { "counters": { "filters_active": 0, "rows": 0 }, - "duration_ms": 0.1858, + "duration_ms": 0.2773, "name": "lexical.filtered_sql", "parent": 12, - "start_ms": 1.0362 + "start_ms": 2.1576999999999997 }, { "counters": {}, - "duration_ms": 0.27109999999999995, + "duration_ms": 0.146, "name": "lexical.media_metadata", "parent": 1, - "start_ms": 1.2537 + "start_ms": 2.4598 }, { "counters": { "filters_active": 0, "rows": 0 }, - "duration_ms": 0.2419, + "duration_ms": 0.112, "name": "lexical.filtered_sql", "parent": 14, - "start_ms": 1.2715999999999998 + "start_ms": 2.4749 }, { "counters": {}, "duration_ms": 0.0155, "name": "hybrid.identifier_priority", "parent": 0, - "start_ms": 1.5598999999999998 + "start_ms": 2.6382 }, { "counters": { "failed": 0, - "request_encode_us": 20, - "request_write_us": 54, - "response_validate_us": 94, - "worker_batch_prepare_us": 16, - "worker_inference_us": 77535, - "worker_pool_normalize_us": 18, - "worker_token_count": 13, - "worker_tokenization_us": 146, - "worker_wait_and_response_decode_us": 78781 + "query_cache_hit": 0, + "query_cache_miss": 1, + "request_encode_us": 18, + "request_write_us": 58, + "response_validate_us": 69, + "worker_batch_prepare_us": 19, + "worker_inference_us": 87311, + "worker_pool_normalize_us": 14, + "worker_token_count": 16, + "worker_tokenization_us": 173, + "worker_wait_and_response_decode_us": 88454 }, - "duration_ms": 79.06530000000001, + "duration_ms": 89.1821, "name": "semantic.query_embedding", "parent": 0, - "start_ms": 1.5843999999999998 + "start_ms": 2.6649 }, { "counters": { "general_lane": 1 }, - "duration_ms": 0.8277, + "duration_ms": 0.8757, "name": "semantic.routing", "parent": 0, - "start_ms": 80.6811 + "start_ms": 91.8792 }, { "counters": { @@ -255,99 +257,99 @@ "filesystem_checks": 2, "result_files": 7 }, - "duration_ms": 0.7831, + "duration_ms": 0.8334, "name": "semantic.retrieval", "parent": 18, - "start_ms": 80.71549999999999 + "start_ms": 91.90950000000001 }, { "counters": {}, - "duration_ms": 0.1451, + "duration_ms": 0.1384, "name": "semantic.generation_snapshot", "parent": 19, - "start_ms": 80.73429999999999 + "start_ms": 91.9247 }, { "counters": { "cache_hit": 1 }, - "duration_ms": 0.017499999999999998, + "duration_ms": 0.0115, "name": "semantic.ann_load", "parent": 19, - "start_ms": 80.89150000000001 + "start_ms": 92.07310000000001 }, { "counters": {}, - "duration_ms": 0.0108, + "duration_ms": 0.011, "name": "semantic.filter_normalization", "parent": 19, - "start_ms": 80.9172 + "start_ms": 92.0933 }, { "counters": { "cache_hit": 1, "eligible_chunks": 8 }, - "duration_ms": 0.0245, + "duration_ms": 0.0252, "name": "semantic.eligibility", "parent": 19, - "start_ms": 80.93599999999999 + "start_ms": 92.1129 }, { "counters": { "requested_chunks": 8 }, - "duration_ms": 0.49450000000000005, + "duration_ms": 0.5494, "name": "semantic.ann_pass", "parent": 19, - "start_ms": 80.9745 + "start_ms": 92.1485 }, { "counters": { "returned_chunks": 8 }, - "duration_ms": 0.036000000000000004, + "duration_ms": 0.0347, "name": "semantic.ann_search", "parent": 24, - "start_ms": 80.9829 + "start_ms": 92.161 }, { "counters": { "filesystem_checks": 2, "unique_parents": 7 }, - "duration_ms": 0.4331, + "duration_ms": 0.4802, "name": "semantic.parent_materialization", "parent": 24, - "start_ms": 81.0271 + "start_ms": 92.2066 }, { "counters": { "valid": 1 }, - "duration_ms": 0.1684, + "duration_ms": 0.20550000000000002, "name": "semantic.parent_filesystem_validation", "parent": 26, - "start_ms": 81.1383 + "start_ms": 92.30539999999999 }, { "counters": { "valid": 1 }, - "duration_ms": 0.0964, + "duration_ms": 0.1146, "name": "semantic.parent_filesystem_validation", "parent": 26, - "start_ms": 81.333 + "start_ms": 92.5374 }, { "counters": { "lexical_files": 0, "semantic_files": 7 }, - "duration_ms": 0.0385, + "duration_ms": 0.0376, "name": "hybrid.rank_fusion", "parent": 0, - "start_ms": 81.5188 + "start_ms": 92.7663 } ] }, @@ -355,18 +357,13 @@ "first_source": { "kind": "page", "number": 2 - }, - "ui_checks": { - "literal_query_marks": true, - "actual_page_explanation": true, - "why_disclosure_collapsed_by_default": true } } ], "screenshots": [ { "file": "search-light.png", - "sha256": "a8347729d7995d0354fd45c1d0762d3fe33adbf0dfcf9ac3387073e522710b15", + "sha256": "bd45041de13093f7a7e26285618f4c1afc5b081fb41e51ff06e1313216af3562", "query": "cancel the supplier partnership", "mode": "basic", "result_count": 7, @@ -374,7 +371,7 @@ "first_source_kind": "page", "first_source_number": 2, "theme": "light", - "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 85 MS · SHOWN 543 MS", + "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 2.1 MS · SHOWN 233 MS", "state_assertions": { "semantic_ready": true, "hybrid_final": true, @@ -387,7 +384,7 @@ }, { "file": "search-dark.png", - "sha256": "b8cf9505ada4afb439edd469d58868fff92dd99cf9b62b6d4a5127307c1e36a7", + "sha256": "cd7e2aa3f7027cd9b40965171a232f6ee901e1e2b971f04628b4cfcc3b440263", "query": "cancel the supplier partnership", "mode": "basic", "result_count": 7, @@ -395,7 +392,7 @@ "first_source_kind": "page", "first_source_number": 2, "theme": "dark", - "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 85 MS · SHOWN 543 MS", + "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 2.1 MS · SHOWN 233 MS", "state_assertions": { "semantic_ready": true, "hybrid_final": true, @@ -408,7 +405,7 @@ }, { "file": "media-search.png", - "sha256": "db832104e3b2fc59bd07a4ea393d38d363c00385023e5f9e469726e076e9b9fe", + "sha256": "4c278446963cd5dbad073bea93a3ca33fb9c10cb43a9544258d5fb6039014d81", "query": "mountains reflected in calm water", "mode": "basic", "result_count": 7, @@ -416,7 +413,7 @@ "first_source_kind": "image", "first_source_number": 0, "theme": "light", - "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 84 MS · SHOWN 519 MS", + "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 1.6 MS · SHOWN 217 MS", "state_assertions": { "semantic_ready": true, "hybrid_final": true, @@ -427,23 +424,20 @@ }, { "file": "performance-details.png", - "sha256": "149753ca90cda5d81254d43e15118b61325a61d7f0fb366b783d4e2dfac5d96d", - "query": "cancel the supplier partnership", + "sha256": "4f336d4c42d0c4a50ce33bca27595f02da7d896c8734d447350d50dab5c339df", + "query": "how can we cancel the supplier partnership", "mode": "debug", "result_count": 7, "selected_filename": "Supplier Agreement.pdf", "first_source_kind": "page", "first_source_number": 2, "theme": "light", - "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 82 MS · SHOWN 514 MS", + "summary": "7 RESULTS · KEYWORD + MEANING · SEARCH 93 MS · SHOWN 311 MS", "state_assertions": { "semantic_ready": true, "hybrid_final": true, "image_visible": false, - "actual_worker_inference": true, - "literal_query_marks": true, - "actual_page_explanation": true, - "why_disclosure_collapsed_by_default": true + "actual_worker_inference": true } } ] diff --git a/docs/screenshots/media-search.png b/docs/screenshots/media-search.png index 13cf28a..6e879b3 100644 Binary files a/docs/screenshots/media-search.png and b/docs/screenshots/media-search.png differ diff --git a/docs/screenshots/performance-details.png b/docs/screenshots/performance-details.png index abb1616..c959060 100644 Binary files a/docs/screenshots/performance-details.png and b/docs/screenshots/performance-details.png differ diff --git a/docs/screenshots/search-dark.png b/docs/screenshots/search-dark.png index d466ce4..462fc7a 100644 Binary files a/docs/screenshots/search-dark.png and b/docs/screenshots/search-dark.png differ diff --git a/docs/screenshots/search-light.png b/docs/screenshots/search-light.png index 63720d2..324df42 100644 Binary files a/docs/screenshots/search-light.png and b/docs/screenshots/search-light.png differ diff --git a/tools/desktop-capture/capture.mjs b/tools/desktop-capture/capture.mjs index dc16744..c217810 100644 --- a/tools/desktop-capture/capture.mjs +++ b/tools/desktop-capture/capture.mjs @@ -306,10 +306,13 @@ try { { timeout: 60000 }, ); await screenshot("media-search", imageQuery); - await page.locator("#settings summary").click(); + await page.locator("#settings > summary").click(); await page.locator("#performance-toggle").check(); - await page.locator("#settings summary").click(); - await query(documentQuery, true); + await page.locator("#settings > summary").click(); + // Use a genuinely unseen query so this screenshot records real inference, + // rather than attributing a bounded query-vector cache hit to the worker. + const performanceQuery = "how can we cancel the supplier partnership"; + await query(performanceQuery, true); await page.locator("#performance-panel").waitFor({ state: "visible" }); await page.locator("#performance-panel summary").click(); await page.waitForFunction( @@ -332,7 +335,7 @@ try { .filter({ hasText: "worker inference" }) .first() .scrollIntoViewIfNeeded(); - await screenshot("performance-details", documentQuery); + await screenshot("performance-details", performanceQuery); const semanticResponse = await context.request.post( new URL("/capture-api/semantic_status", target).href, { diff --git a/tools/embeddinggemma2/benchmark.py b/tools/embeddinggemma2/benchmark.py new file mode 100644 index 0000000..a127b0f --- /dev/null +++ b/tools/embeddinggemma2/benchmark.py @@ -0,0 +1,248 @@ +"""Explicit CPU-only measurements of the real verified Searchi Google runtime. + +Native mode isolates worker IPC/inference. MCP mode measures the actual shared +service over stdio against an explicitly supplied, already indexed test corpus. +No models are downloaded, files indexed, or production defaults modified. +""" +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import queue +import statistics +import subprocess +import threading +import time + +ROOT = Path(__file__).resolve().parents[2] +QUERIES = [ + 'cancel the supplier partnership', + 'mountains reflected in calm water', + 'die Kündigungsfrist im Lieferantenvertrag', + 'source code that checks indexed file permissions', + 'music with piano and a slow melody', + 'a boat crossing a lake at sunset', + 'notes about returning unused materials', + 'find the recording discussing orange soil', +] + +def elapsed(start): + return (time.perf_counter() - start) * 1000 + +def distribution(values): + ordered = sorted(values) + return {'samples': len(values), 'mean_ms': statistics.mean(values), + 'p50_ms': statistics.median(values), + 'p95_ms': ordered[max(0, math.ceil(len(values) * .95) - 1)]} + +class Session: + def __init__(self, command): + environment = os.environ.copy() + environment['SEARCHI_FORCE_CPU'] = '1' + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + self.started = time.perf_counter() + self.process = subprocess.Popen(command, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding='utf-8', env=environment, creationflags=flags) + self.lines = queue.Queue() + self.errors = [] + threading.Thread(target=self.read, daemon=True).start() + threading.Thread(target=self.read_errors, daemon=True).start() + + def read(self): + try: + for line in self.process.stdout: + self.lines.put(json.loads(line)) + self.lines.put(RuntimeError('Runtime stdout closed')) + except Exception as error: + self.lines.put(error) + + def read_errors(self): + for line in self.process.stderr: + if len(self.errors) < 20: + self.errors.append(line.rstrip()) + + def next(self): + result = self.lines.get(timeout=120) + if isinstance(result, Exception): + raise result + return result + + def send(self, value): + self.process.stdin.write(json.dumps(value, ensure_ascii=False) + '\n') + self.process.stdin.flush() + + def worker(self, value): + start = time.perf_counter() + self.send(value) + response = self.next() + if not response.get('ok'): + raise RuntimeError(response.get('error', 'Worker request failed')) + assert response['device'] == 'cpu' + return response, elapsed(start) + + def mcp(self, identifier, method, params): + start = time.perf_counter() + self.send({'jsonrpc': '2.0', 'id': identifier, 'method': method, 'params': params}) + while True: + response = self.next() + if response.get('id') == identifier: + assert 'error' not in response, response + return response['result'], elapsed(start) + + def close(self): + self.process.stdin.close() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=10) + +def vectors(response, count): + result = response['vectors'] + assert len(result) == count + for vector in result: + assert len(vector) == 768 and all(math.isfinite(n) for n in vector) + assert abs(sum(n*n for n in vector) - 1) < .0001 + return result + +def native(args): + manifest = json.loads((ROOT / 'tools/embeddinggemma2/model-manifest.json').read_text()) + model = args.models / manifest['files'][0]['path'] + start = time.perf_counter() + assert model.stat().st_size == manifest['files'][0]['bytes'] + with model.open('rb') as stream: + assert hashlib.file_digest(stream, 'sha256').hexdigest() == manifest['files'][0]['sha256'] + verification_ms = elapsed(start) + reports = [] + for threads in args.threads: + session = Session([str(args.worker.resolve()), '--model', str(model.resolve()), + '--threads', str(threads), '--device', 'cpu']) + try: + hello = session.next() + load_ms = elapsed(session.started) + assert hello['device'] == 'cpu' and hello['model'] == manifest['model_id'] + first, first_ms = session.worker({'op':'query', 'text':'first unseen foreground query', + 'debug_performance':True}) + vectors(first, 1) + samples = [] + for repeat in range(args.repeats): + for query in QUERIES: + response, duration = session.worker({'op':'query','text':query, + 'debug_performance':True}) + vectors(response, 1) + samples.append({'query':query,'repeat':repeat,'round_trip_ms':duration, + 'worker':response['performance']}) + window_samples = [] + passage = ('Supplier cancellation requires sixty days written notice. ' + 'Kündigung und Rückgabe der Materialien. ') * 500 + for _ in range(3): + response, duration = session.worker({'op':'windows','text':passage,'title':'Agreement'}) + encoded = passage.encode() + for window in response['windows']: + assert encoded[window['start_byte']:window['end_byte']].decode() == window['text'] + assert response['windows'][-1]['end_byte'] == len(encoded) + window_samples.append({'bytes':len(encoded),'windows':len(response['windows']), + 'round_trip_ms':duration}) + documents = [(f'Document {n}. Supplier cancellation requires sixty days written notice. ' + 'Unused materials must be returned. ') * 5 for n in range(8)] + singles = [] + single_times = [] + for document in documents: + response, duration = session.worker({'op':'documents','texts':[document],'title':'Agreement'}) + singles.extend(vectors(response, 1)) + single_times.append(duration) + response, batch_ms = session.worker({'op':'documents','texts':documents,'title':'Agreement'}) + batched = vectors(response, 8) + cosines = [sum(a*b for a,b in zip(left,right)) for left,right in zip(singles,batched)] + assert min(cosines) > .99999 + reports.append({'threads':threads,'worker_info':hello,'process_load_ms':load_ms, + 'first_query_ms':first_ms,'first_query_worker':first['performance'], + 'warm_queries':distribution([sample['round_trip_ms'] for sample in samples]), + 'query_samples':samples,'window_samples':window_samples, + 'documents':{'single_request_ms':single_times,'eight_single_total_ms':sum(single_times), + 'eight_document_request_ms':batch_ms,'batch_cosine_min':min(cosines)}}) + print(f'CPU {threads} threads: warm p95 {reports[-1]["warm_queries"]["p95_ms"]:.2f} ms', flush=True) + finally: + session.close() + return {'text_model_verification_ms':verification_ms,'thread_reports':reports, + 'scope':'Native actual Google Q8 worker; verified model; cached OS pages possible. ' + 'Includes NDJSON encode/decode; excludes Rust adapter, ANN and UI. ' + 'First query precedes warm samples. Documents measure IPC batch, not assumed parallelism.'} + +def mcp(args): + assert args.db and args.cli + assert args.db.is_file() + session = Session([str(args.cli.resolve()),'mcp','--db',str(args.db.resolve()), + '--model-dir',str(args.models.resolve()),'--embedding-worker',str(args.worker.resolve()), + '--cpu-threads','1']) + try: + session.mcp(1,'initialize',{'protocolVersion':'2025-11-25','capabilities':{}, + 'clientInfo':{'name':'searchi-cpu-performance','version':'1'}}) + startup_ms = elapsed(session.started) + session.send({'jsonrpc':'2.0','method':'notifications/initialized'}) + stats,_ = session.mcp(2,'tools/call',{'name':'stats','arguments':{}}) + assert stats['structuredContent']['mode'] == 'hybrid' + assert stats['structuredContent']['worker']['device'] == 'cpu' + samples=[] + identifier=3 + cases = [('first_unseen',QUERIES[0],None)] + cases += [('same_query',QUERIES[0],None)]*args.repeats + cases += [('filter_change',QUERIES[0],ext) for ext in ['pdf','md','pdf','md']] + cases += [('distinct_query',query,None) for query in QUERIES[1:]] + for case,query,extension in cases: + arguments={'query':query,'limit':3} + if extension: + arguments['extension']=extension + result,duration=session.mcp(identifier,'tools/call',{'name':'search','arguments':arguments}) + identifier+=1 + assert not result.get('isError'), result + response=result['structuredContent'] + assert response['mode']=='hybrid' + if extension: + assert all(item['filename'].lower().endswith('.'+extension) for item in response['results']) + samples.append({'case':case,'query':query,'extension':extension,'request_ms':duration, + 'engine_ms':response['metrics']['total_ms'], + 'matches':[{'filename':item['filename'],'source':item['source'], + 'snippet':item['snippet'],'score':item['score']} + for item in response['results']]}) + return {'startup_to_initialized_ms':startup_ms,'stats':stats['structuredContent'], + 'samples':samples,'same_query':distribution([s['engine_ms'] for s in samples if s['case']=='same_query']), + 'filter_change':distribution([s['engine_ms'] for s in samples if s['case']=='filter_change']), + 'scope':'Actual shared service through real MCP. Startup includes verified models and background warmup. ' + 'Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint.'} + finally: + session.close() + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--models',type=Path,required=True) + parser.add_argument('--worker',type=Path,required=True) + parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--mode',choices=['native','mcp'],default='native') + parser.add_argument('--threads',type=int,nargs='+',default=[1,2,4]) + parser.add_argument('--repeats',type=int,default=3) + parser.add_argument('--db',type=Path) + parser.add_argument('--cli',type=Path) + parser.add_argument('--rust-profile',choices=['debug','release','not-applicable'],default='not-applicable') + parser.add_argument('--label',default='') + args=parser.parse_args() + assert 1<=args.repeats<=20 and all(1<=n<=8 for n in args.threads) + def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream,'sha256').hexdigest() + report={'captured_at_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()), + 'os':platform.platform(),'machine':platform.machine(),'logical_cpus':os.cpu_count(), + 'cpu_only':True,'rust_profile':args.rust_profile,'label':args.label, + 'worker_sha256':digest(args.worker), + 'cli_sha256':digest(args.cli) if args.cli else None, + 'results':native(args) if args.mode=='native' else mcp(args)} + args.output.parent.mkdir(parents=True,exist_ok=True) + args.output.write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n',encoding='utf-8') + print(f'Saved {args.output}',flush=True) + +if __name__=='__main__': + main()