Files
searchi/docs/evidence/embeddinggemma2-mcp-before.json
karl 8b9b079469
Core CI / core (ubuntu-latest) (push) Failing after 3m43s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 5m56s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
perf: audit EmbeddingGemma 2 and reuse measured query embeddings
2026-10-12 20:59:10 +02:00

733 lines
28 KiB
JSON

{
"captured_at_utc": "2026-10-12T18:35:22Z",
"os": "Windows-11-10.0.26300-SP0",
"machine": "AMD64",
"logical_cpus": 4,
"cpu_only": true,
"rust_profile": "debug",
"results": {
"startup_to_initialized_ms": 7216.115900024306,
"stats": {
"files": {
"content_files": 5,
"files": 7
},
"mode": "hybrid",
"semantic_available": true,
"worker": {
"device": "cpu",
"dimensions": 768,
"media_supported": true,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
}
},
"samples": [
{
"case": "first_unseen",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 85.7454999932088,
"engine_ms": 82.9858,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 84.06170000671409,
"engine_ms": 82.0945,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 69.04040000517853,
"engine_ms": 67.36319999999999,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 66.24039998860098,
"engine_ms": 64.4344,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 67.91299997712485,
"engine_ms": 66.0718,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 70.40650001727045,
"engine_ms": 68.77619999999999,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 79.71600000746548,
"engine_ms": 77.87230000000001,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 69.87529998878017,
"engine_ms": 68.0868,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "same_query",
"query": "cancel the supplier partnership",
"extension": null,
"request_ms": 77.02430000063032,
"engine_ms": 74.836,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "filter_change",
"query": "cancel the supplier partnership",
"extension": "pdf",
"request_ms": 84.02040001237765,
"engine_ms": 79.5709,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
}
]
},
{
"case": "filter_change",
"query": "cancel the supplier partnership",
"extension": "md",
"request_ms": 72.45289999991655,
"engine_ms": 70.70519999999999,
"matches": [
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.01639344262295082
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.016129032258064516
},
{
"filename": "Travel Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
"score": 0.015873015873015872
}
]
},
{
"case": "filter_change",
"query": "cancel the supplier partnership",
"extension": "pdf",
"request_ms": 68.47949998336844,
"engine_ms": 66.9529,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
}
]
},
{
"case": "filter_change",
"query": "cancel the supplier partnership",
"extension": "md",
"request_ms": 68.12429998535663,
"engine_ms": 66.5463,
"matches": [
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.01639344262295082
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.016129032258064516
},
{
"filename": "Travel Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "mountains reflected in calm water",
"extension": null,
"request_ms": 74.91510000545532,
"engine_ms": 73.53999999999999,
"matches": [
{
"filename": "Mountain Lake.jpg",
"source": {
"kind": "image",
"number": 0
},
"snippet": "Image content",
"score": 0.01639344262295082
},
{
"filename": "Night Pier.jpg",
"source": {
"kind": "image",
"number": 0
},
"snippet": "Image content",
"score": 0.016129032258064516
},
{
"filename": "Travel Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "die Kündigungsfrist im Lieferantenvertrag",
"extension": null,
"request_ms": 84.24939998076297,
"engine_ms": 82.6408,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.016129032258064516
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "source code that checks indexed file permissions",
"extension": null,
"request_ms": 79.35359998373315,
"engine_ms": 77.66290000000001,
"matches": [
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.01639344262295082
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.016129032258064516
},
{
"filename": "Night Pier.jpg",
"source": {
"kind": "image",
"number": 0
},
"snippet": "Image content",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "music with piano and a slow melody",
"extension": null,
"request_ms": 80.26409999001771,
"engine_ms": 78.6324,
"matches": [
{
"filename": "Travel Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
"score": 0.01639344262295082
},
{
"filename": "Night Pier.jpg",
"source": {
"kind": "image",
"number": 0
},
"snippet": "Image content",
"score": 0.016129032258064516
},
{
"filename": "Garden Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "a boat crossing a lake at sunset",
"extension": null,
"request_ms": 79.87940000020899,
"engine_ms": 78.0203,
"matches": [
{
"filename": "Night Pier.jpg",
"source": {
"kind": "image",
"number": 0
},
"snippet": "Image content",
"score": 0.01639344262295082
},
{
"filename": "Mountain Lake.jpg",
"source": {
"kind": "image",
"number": 0
},
"snippet": "Image content",
"score": 0.016129032258064516
},
{
"filename": "Travel Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Travel plans\nTake the train to Berlin for the museum weekend. Reserve a room near the station and pack a raincoat.",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "notes about returning unused materials",
"extension": null,
"request_ms": 73.2068000070285,
"engine_ms": 71.48490000000001,
"matches": [
{
"filename": "Supplier Agreement.pdf",
"source": {
"kind": "page",
"number": 2
},
"snippet": "Termination and cancellation The supplier partnership may be ended with sixty days written notice. Send the cancellation letter to the vendor. Return unused materials and settle outstanding invoices. \n",
"score": 0.01639344262295082
},
{
"filename": "Garden Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
"score": 0.016129032258064516
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.015873015873015872
}
]
},
{
"case": "distinct_query",
"query": "find the recording discussing orange soil",
"extension": null,
"request_ms": 79.31890001054853,
"engine_ms": 77.6251,
"matches": [
{
"filename": "Garden Notes.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Garden notes\nTomatoes need sunlight and regular watering. Plant herbs in pots on the balcony and add compost in spring.",
"score": 0.01639344262295082
},
{
"filename": "Q3 Procurement.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Q3 procurement review\nWe will review supplier delivery times, packaging costs and the annual purchasing budget. Present the comparison of existing vendors at Thursday's planning meeting.",
"score": 0.016129032258064516
},
{
"filename": "Vendor Onboarding.md",
"source": {
"kind": "line",
"number": 1
},
"snippet": "# Vendor onboarding\nCollect supplier tax details, banking information and insurance documents. The purchasing team completes compliance review before signing a new partnership.",
"score": 0.015873015873015872
}
]
}
],
"same_query": {
"samples": 8,
"mean_ms": 71.1919,
"p50_ms": 68.4315,
"p95_ms": 82.0945
},
"filter_change": {
"samples": 4,
"mean_ms": 70.943825,
"p50_ms": 68.82905,
"p95_ms": 79.5709
},
"scope": "Actual shared service through real MCP. Startup includes verified models and background warmup. Seven-file README corpus when using that explicit fixture DB; no UI scheduling or paint."
},
"label": "baseline-no-query-vector-cache",
"worker_sha256": "1c3bd856e56f49a1ffc11ec3eeaef20297bda5f1539427ef772dc9045e7da8b3",
"cli_sha256": null,
"measurement_base_commit": "9870df7",
"metadata_note": "Worker unchanged during these runs; hash recorded afterward. Baseline CLI executable hash was not captured; source checkpoint was 9870df7."
}