karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

feat: add multimodal search and granular performance diagnostics
Core CI / core (ubuntu-latest) (push) Failing after 3m31s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 5m39s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s

Browse Source
This commit is contained in:
karl committed 2026-10-12 19:16:23 +02:00
1 parent d3bb8c7618
commit b8400c4fbf
110 files changed
+79395 -250
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fixtures/media/reference-summary.json
+502
View File
@@ -0,0 +1,502 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": {
"device": "cpu",
"dimensions": 768,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"overall": {
"positive_queries": 55,
"unsupported_queries": 4,
"recall_at1": 0.9636363636363636,
"recall_at5": 0.9636363636363636,
"mrr": 0.9636363636363636,
"temporal_queries": 22,
"segment_overlap_hit_at1": 0.9545454545454546,
"timestamp_point_hit_at1": 0.6818181818181818,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 6,
"no_answer_return_rate": 1.0
},
"by_category": {
"content-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"exact-filter": {
"positive_queries": 1,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"exact-metadata": {
"positive_queries": 0,
"unsupported_queries": 3,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"filter-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"image-memory": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"metadata-no-answer": {
"positive_queries": 0,
"unsupported_queries": 1,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"music-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 0.5,
"recall_at5": 0.5,
"mrr": 0.5,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"sound-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"speech-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"temporal-negative": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"video-action": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-scene": {
"positive_queries": 6,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 6,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-speech": {
"positive_queries": 8,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 8,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 0.25,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 13.129,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-temporal": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 0.75,
"timestamp_point_hit_at1": 0.75,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 1.0,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
"by_language": {
"de": {
"positive_queries": 27,
"unsupported_queries": 0,
"recall_at1": 0.9629629629629629,
"recall_at5": 0.9629629629629629,
"mrr": 0.9629629629629629,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.9090909090909091,
"timestamp_point_hit_at1": 0.6363636363636364,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"en": {
"positive_queries": 28,
"unsupported_queries": 4,
"recall_at1": 0.9642857142857143,
"recall_at5": 0.9642857142857143,
"mrr": 0.9642857142857143,
"temporal_queries": 11,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 0.7272727272727273,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 4,
"no_answer_return_rate": 1.0
}
},
"encoding_failures": [
{
"candidate": {
"asset": "music02",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\music02-audio-0000.f32",
"start_s": 0
},
"error": "Invalid normalized audio PCM"
}
],
"encoding_ms": 498657.0532,
"query_p50_ms": 92.78150000000001,
"query_p95_ms": 122.9277,
"failures": [
{
"query": "q030",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4,
"modality": "image",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip03-frame-0002.png",
"start_s": 2,
"token_budget": 70
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.0
},
{
"query": "q039",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q040",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q041",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": 13.129
},
{
"query": "q042",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": 13.129
},
{
"query": "q043",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 60,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
"start_s": 30
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 13.777000000000001
},
{
"query": "q044",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 60,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
"start_s": 30
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 13.777000000000001
},
{
"query": "q049",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q050",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q055",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
},
{
"query": "q056",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
},
{
"query": "q057",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
},
{
"query": "q059",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip01",
"score": 0.750250822387441
},
{
"query": "q060",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.5716409940327322
},
{
"query": "q061",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.6223961635527826
},
{
"query": "q062",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.6181057666050869
},
{
"query": "q063",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.6229413326862353
},
{
"query": "q064",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip03",
"score": 0.5715326739972073
},
{
"query": "q065",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
}
],
"source_scope": "Direct real-model dense reference baseline; candidate filters here are benchmark reference constraints, not production-filter certification; no rejection threshold is tuned",
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}