267 lines
6.7 KiB
JSON
267 lines
6.7 KiB
JSON
|
{
|
||
|
|
"recorded_at": "2026-10-12T19:19:20.376773+00:00",
|
||
|
|
"device": "cpu",
|
||
|
|
"cpu_threads": 1,
|
||
|
|
"model": "google/embeddinggemma-2",
|
||
|
|
"precision": "verified Q8_0 weights, CPU inference",
|
||
|
|
"query_task": "task: code retrieval | query: ",
|
||
|
|
"scope": "8 tiny source files and 1 ordinary-document hard negative; synthetic regression, not general code accuracy",
|
||
|
|
"fixture_sha256": {
|
||
|
|
"a.py": "78d878583e1e29f8a33373e40e2541fef00ae706e1b01dd52b98e8a802e9def6",
|
||
|
|
"b.cpp": "9cdf9ba0abd4a7d068e37405bf40405c6f60cefa0cf83f072efb7419091acd39",
|
||
|
|
"c.ts": "6a23062539b6944767195506b086e7f0071e68ab636369a917efe4794abb61b4",
|
||
|
|
"d.py": "f2f221eab12c05e1a2da45c683e55d0d3bc8c934ebf393365b026ea181381b05",
|
||
|
|
"e.cs": "d05a8a3c8471263c9fc8ec940f482531a671b81cc518de6e43ed1e2c57fa3ee6",
|
||
|
|
"f.php": "41036ba20c6019cb94b7f7fde540f69449859ae19a6786751a6cfeec2880797e",
|
||
|
|
"g.py": "7bce1fb7efe687ad71ba63285a83d38fb322acdd0266d3ea59b25faaf96485c9",
|
||
|
|
"h.rs": "692577a259c1fafbc4285a63968294f05818944f2cd7aa17b0eb31b9b6254649",
|
||
|
|
"notes.txt": "2ab4d304fc6b0e03896a31cc409c27b0d458c172f6ffbc32960291c7fbd13f1f"
|
||
|
|
},
|
||
|
|
"queries": [
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "calculate a file's SHA256 without loading the entire file into memory",
|
||
|
|
"expected": "a.py",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 8.568600023863837,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "find the first position where sorted numbers are greater than or equal to a target",
|
||
|
|
"expected": "b.cpp",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.721800013910979,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "sum transaction amounts separately for each category",
|
||
|
|
"expected": "c.ts",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.4463999832514673,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "retry failed requests with exponential backoff",
|
||
|
|
"expected": "d.py",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.4552999928127974,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "release a semaphore even if the asynchronous operation throws",
|
||
|
|
"expected": "e.cs",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.4591999934054911,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "escape unsafe HTML characters before displaying user input",
|
||
|
|
"expected": "f.php",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.43420000676997,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "return the ten most expensive records",
|
||
|
|
"expected": "g.py",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.2260999938007444,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "Datei prüfen ohne sie vollständig in den Speicher zu laden",
|
||
|
|
"expected": "a.py",
|
||
|
|
"rank": null,
|
||
|
|
"results": [],
|
||
|
|
"roundtrip_ms": 1.4165000175125897,
|
||
|
|
"source_verified": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "lexical_only",
|
||
|
|
"query": "refreshAccessToken",
|
||
|
|
"expected": "h.rs",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"h.rs"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 1.4574000088032335,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "calculate a file's SHA256 without loading the entire file into memory",
|
||
|
|
"expected": "a.py",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"a.py",
|
||
|
|
"e.cs",
|
||
|
|
"d.py",
|
||
|
|
"h.rs",
|
||
|
|
"c.ts",
|
||
|
|
"b.cpp",
|
||
|
|
"f.php",
|
||
|
|
"g.py"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 228.21230001864024,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "find the first position where sorted numbers are greater than or equal to a target",
|
||
|
|
"expected": "b.cpp",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"b.cpp",
|
||
|
|
"g.py",
|
||
|
|
"c.ts",
|
||
|
|
"e.cs",
|
||
|
|
"d.py",
|
||
|
|
"a.py",
|
||
|
|
"h.rs",
|
||
|
|
"f.php"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 143.91000001342036,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "sum transaction amounts separately for each category",
|
||
|
|
"expected": "c.ts",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"c.ts",
|
||
|
|
"g.py",
|
||
|
|
"e.cs",
|
||
|
|
"h.rs",
|
||
|
|
"f.php",
|
||
|
|
"a.py",
|
||
|
|
"d.py",
|
||
|
|
"b.cpp"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 109.81110000284389,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "retry failed requests with exponential backoff",
|
||
|
|
"expected": "d.py",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"d.py",
|
||
|
|
"h.rs",
|
||
|
|
"e.cs",
|
||
|
|
"g.py",
|
||
|
|
"a.py",
|
||
|
|
"c.ts",
|
||
|
|
"f.php",
|
||
|
|
"b.cpp"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 126.85969998710789,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "release a semaphore even if the asynchronous operation throws",
|
||
|
|
"expected": "e.cs",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"e.cs",
|
||
|
|
"d.py",
|
||
|
|
"h.rs",
|
||
|
|
"f.php",
|
||
|
|
"a.py",
|
||
|
|
"c.ts",
|
||
|
|
"g.py",
|
||
|
|
"b.cpp"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 121.69749999884516,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "escape unsafe HTML characters before displaying user input",
|
||
|
|
"expected": "f.php",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"f.php",
|
||
|
|
"d.py",
|
||
|
|
"e.cs",
|
||
|
|
"h.rs",
|
||
|
|
"c.ts",
|
||
|
|
"a.py",
|
||
|
|
"g.py",
|
||
|
|
"b.cpp"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 100.53349999361672,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "return the ten most expensive records",
|
||
|
|
"expected": "g.py",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"g.py",
|
||
|
|
"c.ts",
|
||
|
|
"d.py",
|
||
|
|
"h.rs",
|
||
|
|
"e.cs",
|
||
|
|
"a.py",
|
||
|
|
"f.php",
|
||
|
|
"b.cpp"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 85.61569999437779,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "Datei prüfen ohne sie vollständig in den Speicher zu laden",
|
||
|
|
"expected": "a.py",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"a.py",
|
||
|
|
"b.cpp",
|
||
|
|
"e.cs",
|
||
|
|
"d.py",
|
||
|
|
"h.rs",
|
||
|
|
"g.py",
|
||
|
|
"f.php",
|
||
|
|
"c.ts"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 110.75749999145046,
|
||
|
|
"source_verified": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mode": "hybrid",
|
||
|
|
"query": "refreshAccessToken",
|
||
|
|
"expected": "h.rs",
|
||
|
|
"rank": 1,
|
||
|
|
"results": [
|
||
|
|
"h.rs",
|
||
|
|
"e.cs",
|
||
|
|
"d.py",
|
||
|
|
"c.ts",
|
||
|
|
"a.py",
|
||
|
|
"f.php",
|
||
|
|
"g.py",
|
||
|
|
"b.cpp"
|
||
|
|
],
|
||
|
|
"roundtrip_ms": 63.88430000515655,
|
||
|
|
"source_verified": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|