karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Distinguish main matches from related suggestions and ship 0.1.9

Browse Source
This commit is contained in:
karl committed 2026-10-14 12:45:32 +02:00
1 parent a8ba594d0e
commit b0bdfc7816
40 files changed
+2463 -185
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fixtures/absence-v3/evaluation-summary.json
+233
View File
@@ -0,0 +1,233 @@
{
"report_sha256": "3a032695607a5035016602cbd995d9c4ef028891e5310eb585a9356f2404b628",
"report_path": ".searchi/qa/absence-v3-results.json",
"fixture_sha256": "12354a55c1e5f1efe3437e653c074d247374852a420329db8d500771c4a24f53",
"policy_sha256": "d3d4e437a54d1d62a70bb09c2882db6362a2566525a54f92d5fdc7fdfb91e26c",
"complete": true,
"worker_info": {
"code_query_supported": true,
"context_tokens": 512,
"device": "cpu",
"dimensions": 768,
"media_supported": true,
"model": "google/embeddinggemma-2",
"ready": true,
"role": "query",
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"runtime_hashes": {
"ffmpeg": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46",
"ffprobe": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236",
"model_manifest": "fad82f171f4976d8eeb89c11d6a79fc46d92afb73c10a34f140d5b56e377397f",
"runner": "ca649639de16b8aea021d6cf6766c4240a7dfe54cc173f9ed379538b9c5a3057",
"speech_base": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe",
"speech_vad": "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",
"speech_worker": "9c34aa2b12a4c2ab876a1573045ef9a31e19e7dfff76a2b34efdf45555ad97aa",
"worker": "c01cb3ec13cfca722ee768e8350e1399a8b9a4d14a24a35adc3473a5cfc82a73"
},
"source_hashes": {
"relevance": "de1c031592af13929ddf061da247343e76ecf941b46dfb3821bfce15b9413769",
"routing": "6a7f20ad312517acaf7c9e6120a4eb55edc47f36e19267d54e7053506e01e755",
"service_lib": "12e4fd01149a7755b587b5d7ed9162fbe673d8869241a38feb8b33639fa4ccdc"
},
"asset_count": 36,
"covered_asset_count": 36,
"extraction_errors": [],
"semantic_failures": [],
"speech_failures": [],
"active_chunk_count": 172,
"lexical_ms": 40.328900000000004,
"initial_semantic_index_ms": 894883.7687,
"query_startup_ms": 1017.2243000000001,
"raw_query_median_ms": 94.3792,
"raw_query_max_ms": 122.8708,
"query_order_note": "lexical then raw then hybrid; hybrid reuses query vector. Replay baseline has no separate runtime timing. Scores not confidence; coverage failures must be separated.",
"summary": {
"all": {
"absent_count": 16,
"accessible_absent_nonempty": 16,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 16,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 4,
"positive_count": 16,
"primary_absent_nonempty": 4,
"primary_positive_recall5": 0.625,
"raw_absent_nonempty": 16,
"raw_positive_recall5": 1.0
},
"category:ambiguous_partial": {
"absent_count": 0,
"accessible_absent_nonempty": 0,
"accessible_positive_recall5": null,
"baseline_replay_absent_nonempty": 0,
"baseline_replay_positive_recall5": null,
"partial_count": 4,
"positive_count": 0,
"primary_absent_nonempty": 0,
"primary_positive_recall5": null,
"raw_absent_nonempty": 0,
"raw_positive_recall5": null
},
"category:code": {
"absent_count": 2,
"accessible_absent_nonempty": 2,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 2,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 2,
"primary_absent_nonempty": 2,
"primary_positive_recall5": 1.0,
"raw_absent_nonempty": 2,
"raw_positive_recall5": 1.0
},
"category:document": {
"absent_count": 4,
"accessible_absent_nonempty": 4,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 4,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 4,
"primary_absent_nonempty": 2,
"primary_positive_recall5": 0.5,
"raw_absent_nonempty": 4,
"raw_positive_recall5": 1.0
},
"category:image": {
"absent_count": 2,
"accessible_absent_nonempty": 2,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 2,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 2,
"primary_absent_nonempty": 0,
"primary_positive_recall5": 0.5,
"raw_absent_nonempty": 2,
"raw_positive_recall5": 1.0
},
"category:music": {
"absent_count": 2,
"accessible_absent_nonempty": 2,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 2,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 2,
"primary_absent_nonempty": 0,
"primary_positive_recall5": 0.0,
"raw_absent_nonempty": 2,
"raw_positive_recall5": 1.0
},
"category:sound": {
"absent_count": 2,
"accessible_absent_nonempty": 2,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 2,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 2,
"primary_absent_nonempty": 0,
"primary_positive_recall5": 1.0,
"raw_absent_nonempty": 2,
"raw_positive_recall5": 1.0
},
"category:speech": {
"absent_count": 2,
"accessible_absent_nonempty": 2,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 2,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 2,
"primary_absent_nonempty": 0,
"primary_positive_recall5": 0.5,
"raw_absent_nonempty": 2,
"raw_positive_recall5": 1.0
},
"category:video": {
"absent_count": 2,
"accessible_absent_nonempty": 2,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 2,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 0,
"positive_count": 2,
"primary_absent_nonempty": 0,
"primary_positive_recall5": 1.0,
"raw_absent_nonempty": 2,
"raw_positive_recall5": 1.0
},
"language:de": {
"absent_count": 8,
"accessible_absent_nonempty": 8,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 8,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 2,
"positive_count": 8,
"primary_absent_nonempty": 2,
"primary_positive_recall5": 0.5625,
"raw_absent_nonempty": 8,
"raw_positive_recall5": 1.0
},
"language:en": {
"absent_count": 8,
"accessible_absent_nonempty": 8,
"accessible_positive_recall5": 1.0,
"baseline_replay_absent_nonempty": 8,
"baseline_replay_positive_recall5": 1.0,
"partial_count": 2,
"positive_count": 8,
"primary_absent_nonempty": 2,
"primary_positive_recall5": 0.6875,
"raw_absent_nonempty": 8,
"raw_positive_recall5": 1.0
}
},
"positive_primary_any_relevant_at5": 11,
"positive_primary_top1_correct": 10,
"positive_accessible_top1_correct": 13,
"residual_false_primary": [
{
"id": "a3-007",
"category": "document",
"language": "en",
"primary_asset_ids": [
"museum-a"
],
"top_raw_cosine": 0.7194200754165649
},
{
"id": "a3-008",
"category": "document",
"language": "de",
"primary_asset_ids": [
"museum-a"
],
"top_raw_cosine": 0.7089265584945679
},
{
"id": "a3-011",
"category": "code",
"language": "en",
"primary_asset_ids": [
"code-version"
],
"top_raw_cosine": 0.7125321626663208
},
{
"id": "a3-012",
"category": "code",
"language": "de",
"primary_asset_ids": [
"code-version"
],
"top_raw_cosine": 0.7140055894851685
}
],
"interpretation": "Primary false-neighbor display improved from 16/16 to 4/16 absent queries. Accessible recall@5 stays 1.0; primary mean recall@5 falls to 0.625. Similarity alone cannot prove requested facts exist. No policy parameter was tuned after this holdout. Publisher-based music/sound labels and one AI annotator limit generalization."
}